Oracle ϵͳ - Linux ϵͳ - Backtrack 5 - ȫ - Juniper - Cisco - ˼ģ - Cisco ֤ - Cisco ios
Docker > Ʒ >

ڵλǣDocker > Ʒ > hadoop2.0.4 װ̳

hadoop2.0.4 װ̳

ʱ䣺2018-06-04 13:52  Դδ֪  Ķ Ʒ Ҫ

֮ǰhadoop2.04Իɹһ룬ڲοĵǰ1.0йءIJӢ˵2.05汾İװ

1滮
centOS6.4ϴhadoop2.05java汾7UP21
192.168.100.171        linux1 (namenode)
192.168.100.172        linux2 (Ԥnamenode)
192.168.100.173        linux3 (datanode)
192.168.100.174        linux4 (datanode)
192.168.100.175        linux5 (datanode)

2VMware Workstation
aװcentOS6.4linux1ͨssh,iptables
[root@linux1 ~]# chkconfig sshd on
[root@linux1 ~]# chkconfig iptables off
[root@linux1 ~]# chkconfig ip6tables off
[root@linux1 ~]# chkconfig postfix off

bرlinux1һµӲ̵Ŀ¼ΪӲãʹSCSI1:0ӿ)
޸linux1.vmxӺ޸IJ:
disk.locking="FALSE"    
diskLib.dataCacheMaxSize = "0"
disk.EnableUUID = "TRUE"
scsi1.present = "TRUE"
scsi1.sharedBus = "Virtual"
scsi1.virtualDev = "lsilogic"

clinux1JAVAӲ̣װJAVAڻļ/etc/profileĩβӣ
JAVA_HOME=/usr/java/jdk1.7.0_21; export JAVA_HOME
JRE_HOME=/usr/java/jdk1.7.0_21/jre; export JRE_HOME
CLASSPATH=.:$JAVA_HOME/lib:$JAVA_HOME/lib/tools.jar; export CLASSPATH
PATH=$JAVA_HOME/bin:$JRE_HOME/bin:$PATH; export PATH
************************************************************************
Ϊ˷㣬hadoop/etc/profilehadoopû~/.bashrc
export HADOOP_PREFIX=/app/hadoop205
export PATH=$PATH:$HADOOP_PREFIX/bin
export PATH=$PATH:$HADOOP_PREFIX/sbin
export HADOOP_MAPRED_HOME=${HADOOP_PREFIX}
export HADOOP_COMMON_HOME=${HADOOP_PREFIX}
export HADOOP_HDFS_HOME=${HADOOP_PREFIX}
export YARN_HOME=${HADOOP_PREFIX}
************************************************************************

d޸/etc/hostsӣ
192.168.100.171        linux1
192.168.100.172        linux2
192.168.100.173        linux3
192.168.100.174        linux4
192.168.100.175        linux5

e޸/etc/sysconfig/selinux
SELINUX=disabled

fhadoopûװhadoopļ
[root@linux1 ~]# useradd hadoop -g root
[root@linux1 ~]# passwd hadoop
[root@linux1 ~]# cd /
[root@linux1 /]# mkdir /app
[root@linux1 /]# cd /app
[root@linux1 app]# tar -zxf /mnt/mysoft/LinuxSoft/hadoop-2.0.5-alpha.tar.gz
[root@linux1 app]# mv hadoop-2.0.5-alpha hadoop205
[root@linux1 app]# chown hadoop:root -R /app/hadoop205
[root@linux1 hadoop205]# su - hadoop
[hadoop@linux1 ~]$ cd /app/hadoop205
[hadoop@linux1 hadoop205]$ mkdir mytmp
[hadoop@linux1 hadoop205]$ mkdir mydata
[hadoop@linux1 hadoop205]$ cd mydata
[hadoop@linux1 mydata]$ mkdir name
[hadoop@linux1 mydata]$ mkdir data

g޸hadoopļ

[hadoop@linux1 hadoop205]$ cd etc/hadoop
[hadoop@linux1 hadoop]$ vi core-site.xml
******************************************************************************
<configuration>
<property>
<name>fs.defaultFS</name>
<value>hdfs://linux1:9000</value>
</property>
<property>
<name>io.file.buffer.size</name>
<value>131072</value>
</property>
</configuration>
******************************************************************************

[hadoop@linux1 hadoop]$ vi hdfs-site.xml
******************************************************************************
<configuration>
<property>
<name>dfs.namenode.name.dir</name>
<value>file:/app/hadoop205/mydata/name</value>
</property>

<property>
<name>dfs.blocksize</name>
<value>268435456</value>
</property>

<property>
<name>dfs.namenode.handler.count</name>
<value>100</value>
</property>

<property>
<name>dfs.datanode.data.dir</name>
<value>file:/app/hadoop205/mydata/data</value>
</property>
</configuration>
******************************************************************************

[hadoop@linux1 hadoop]$ vi yarn-site.xml
******************************************************************************
<configuration>
<property>
<name>yarn.resourcemanager.address</name>
<value>linux1:8080</value>
</property>

<property>
<name>yarn.resourcemanager.scheduler.address</name>
<value>linux1:8081</value>
</property>

<property>
<name>yarn.resourcemanager.resource-tracker.address</name>
<value>linux1:8082</value>
</property>

<property>
<name>yarn.resourcemanager.admin.address</name>
<value>linux1:8083</value>
</property>

<property>
<name>yarn.resourcemanager.webapp.address</name>
<value>linux1:8084</value>
</property>

<property>
<name>yarn.nodemanager.aux-services</name>
<value>mapreduce.shuffle</value>
</property>

<property>
<name>yarn.nodemanager.aux-services.mapreduce.shuffle.class</name>
<value>org.apache.hadoop.mapred.ShuffleHandler</value>
</property>
</configuration>
******************************************************************************

[hadoop@linux1 hadoop]$ vi mapred-site.xml
******************************************************************************
<configuration>
<property>
<name>mapreduce.framework.name</name>
<value>yarn</value>
</property>

<property>
<name>mapreduce.map.memory.mb</name>
<value>1536</value>
</property>

<property>
<name>mapreduce.map.java.opts</name>
<value>-Xmx1024M</value>
</property>

<property>
<name>mapreduce.reduce.memory.mb</name>
<value>3072</value>
</property>

<property>
<name>mapreduce.reduce.java.opts</name>
<value>-Xmx2560M</value>
</property>

<property>
<name>mapreduce.task.io.sort.mb</name>
<value>512</value>
</property>

<property>
<name>mapreduce.task.io.sort.factor</name>
<value>100</value>
</property>

<property>
<name>mapreduce.reduce.shuffle.parallelcopies</name>
<value>50</value>
</property>

<property>
<name>mapreduce.job.tracker</name>
<value>hdfs://linux1:9001</value>
<final>true</final>
</property>
</configuration>
******************************************************************************

[hadoop@linux1 hadoop]$ vi hadoop-env.sh
******************************************************************************
export JAVA_HOME=/usr/java/jdk1.7.0_21
export HADOOP_FREFIX=/app/hadoop205
export HADOOP_COMMON_HOME=${HADOOP_FREFIX}
export HADOOP_HDFS_HOME=${HADOOP_FREFIX}
export PATH=$PATH:$HADOOP_FREFIX/bin
export PATH=$PATH:$HADOOP_FREFIX/sbin
export HADOOP_MAPRED_HOME=${HADOOP_FREFIX}
export YARN_HOME=${HADOOP_FREFIX}
export HADOOP_CONF_HOME=${HADOOP_FREFIX}/etc/hadoop
export YARN_CONF_DIR=${HADOOP_FREFIX}/etc/hadoop
******************************************************************************

[hadoop@linux1 hadoop]$ vi yarn-env.sh
******************************************************************************
export JAVA_HOME=/usr/java/jdk1.7.0_21
export HADOOP_FREFIX=/app/hadoop205
export HADOOP_COMMON_HOME=${HADOOP_FREFIX}
export HADOOP_HDFS_HOME=${HADOOP_FREFIX}
export PATH=$PATH:$HADOOP_FREFIX/bin
export PATH=$PATH:$HADOOP_FREFIX/sbin
export HADOOP_MAPRED_HOME=${HADOOP_FREFIX}
export YARN_HOME=${HADOOP_FREFIX}
export HADOOP_CONF_HOME=${HADOOP_FREFIX}/etc/hadoop
export YARN_CONF_DIR=${HADOOP_FREFIX}/etc/hadoop
******************************************************************************

hsshʹ֤֤/etc/ssh/sshd_configעͣ
RSAAuthentication yes
PubkeyAuthentication yes
AuthorizedKeysFile .ssh/authorized_keys


3ssh
aرֱƳlinux2linux3linux4linux5
޸vmware workstationļdisplayname
޸ļصϢ
/etc/udev/rules.d/70-persistent-net.rules
/etc/sysconfig/network
/etc/sysconfig/network-scripts/ifcfg-eth0

blinux1linux2linux3linux4linux5ȷ໥֮pingͨ

csshȷlinux1֤ڵ
[root@linux1 tmp]# su - hadoop
[hadoop@linux1 ~]$ ssh-keygen -t rsa
Generating public/private rsa key pair.
Enter file in which to save the key (/home/hadoop/.ssh/id_rsa): 
Created directory '/home/hadoop/.ssh'.
Enter passphrase (empty for no passphrase): 
Enter same passphrase again: 
Your identification has been saved in /home/hadoop/.ssh/id_rsa.
Your public key has been saved in /home/hadoop/.ssh/id_rsa.pub.
The key fingerprint is:
17:37:98:fa:7e:5c:e4:8b:b4:7e59:28:8f:45:bd hadoop@linux1
The key's randomart image is:
+--[ RSA 2048]----+
|                 |
|           o     |
|          + o    |
|         . o ... |
|        S .  o. .|
|         o  ..o..|
|          .o.+oE.|
|         .  ==oo |
|          .oo.=o |
+-----------------+
[hadoop@linux1 ~]$ ssh-copy-id -i ~/.ssh/id_rsa.pub hadoop@linux1
[hadoop@linux1 ~]$ ssh-copy-id -i ~/.ssh/id_rsa.pub hadoop@linux2
[hadoop@linux1 ~]$ ssh-copy-id -i ~/.ssh/id_rsa.pub hadoop@linux3
[hadoop@linux1 ~]$ ssh-copy-id -i ~/.ssh/id_rsa.pub hadoop@linux4
[hadoop@linux1 ~]$ ssh-copy-id -i ~/.ssh/id_rsa.pub hadoop@linux5

֤ɷʣ
[hadoop@linux1 ~]$ ssh linux1 date
[hadoop@linux1 ~]$ ssh linux2 date
[hadoop@linux1 ~]$ ssh linux3 date
[hadoop@linux1 ~]$ ssh linux4 date
[hadoop@linux1 ~]$ ssh linux5 date

4ʼhadoop
[hadoop@linux1 hadoop205]$ /app/hadoop205/bin/hdfs namenode -format

5linux1slaves
[hadoop@linux1 hadoop205]$ vi etc/hadoop/slaves
192.168.100.173
192.168.100.174
192.168.100.175

6hadoop
[hadoop@linux1 hadoop205]$ /app/hadoop205/sbin/start-all.sh
This script is Deprecated. Instead use start-dfs.sh and start-yarn.sh
13/06/11 10:08:28 WARN util.NativeCodeLoader: Unable to load native-hadoop library for your platform... using builtin-java 

classes where applicable
Starting namenodes on [linux1]
linux1: starting namenode, logging to /app/hadoop205/logs/hadoop-hadoop-namenode-linux1.out
192.168.100.174: starting datanode, logging to /app/hadoop205/logs/hadoop-hadoop-datanode-linux4.out
192.168.100.175: starting datanode, logging to /app/hadoop205/logs/hadoop-hadoop-datanode-linux5.out
192.168.100.173: starting datanode, logging to /app/hadoop205/logs/hadoop-hadoop-datanode-linux3.out
Starting secondary namenodes [0.0.0.0]
0.0.0.0: starting secondarynamenode, logging to /app/hadoop205/logs/hadoop-hadoop-secondarynamenode-linux1.out
13/06/11 10:08:50 WARN util.NativeCodeLoader: Unable to load native-hadoop library for your platform... using builtin-java 

classes where applicable
starting yarn daemons
starting resourcemanager, logging to /app/hadoop205/logs/yarn-hadoop-resourcemanager-linux1.out
192.168.100.174: starting nodemanager, logging to /app/hadoop205/logs/yarn-hadoop-nodemanager-linux4.out
192.168.100.175: starting nodemanager, logging to /app/hadoop205/logs/yarn-hadoop-nodemanager-linux5.out
192.168.100.173: starting nodemanager, logging to /app/hadoop205/logs/yarn-hadoop-nodemanager-linux3.out

7ʵַ
NameNode        http://linux1:50070/        
ResourceManager        http://linux1:8084/
MapReduce JobHistory Server        http://linux1:19888/        ûã
Ѷ