Cluster Installation with Apache Ambari

--Ambari system requirements

https://docs.hortonworks.com/HDPDocuments/Ambari-2.1.2.1/bk_Installing_HDP_AMB/content/_operating_systems_requirements.html

Ambari 1

ec2-10-10-10-10.compute-1.amazonaws.com
ip-20-20-20-20.ec2.internal

Ambari 2

ec2-30-30-30-30.compute-1.amazonaws.com
ip-40-40-40-40.ec2.internal

Ambari 3

ec2-50-50-50-50.compute-1.amazonaws.com
ip-60-60-60-60.ec2.internal

Ambari 4

ec2-70-70-70-70.compute-1.amazonaws.com
ip-80-80-80-80.ec2.internal


--Setup id_rsa on all nodes
cd .ssh
vi id_rsa
sudo chown ubuntu:ubuntu id_rsa
chmod 600 id_rsa


--Copy the downloaded JDK to other instances
scp jdk-8u131-linux-x64.tar.gz ip-40-40-40-40.ec2.internal:/home/ubuntu/
scp jdk-8u131-linux-x64.tar.gz ip-60-60-60-60.ec2.internal:/home/ubuntu/
scp jdk-8u131-linux-x64.tar.gz ip-80-80-80-80.ec2.internal:/home/ubuntu/

--Untar JDK
tar -xvf jdk-8u131-linux-x64.tar.gz

sudo mkdir -p /usr/lib/jvm
sudo mv ./jdk1.8.0_131 /usr/lib/jvm/

sudo update-alternatives --install "/usr/bin/java" "java" "/usr/lib/jvm/jdk1.8.0_131/bin/java" 1
sudo update-alternatives --install "/usr/bin/javac" "javac" "/usr/lib/jvm/jdk1.8.0_131/bin/javac" 1
sudo update-alternatives --install "/usr/bin/javaws" "javaws" "/usr/lib/jvm/jdk1.8.0_131/bin/javaws" 1

--Set permissions
sudo chmod a+x /usr/bin/java 
sudo chmod a+x /usr/bin/javac 
sudo chmod a+x /usr/bin/javaws
sudo chown -R root:root /usr/lib/jvm/jdk1.8.0_131

--Set JAVA_HOME (on all nodes) in /etc/environment
sudo vi /etc/environment
JAVA_HOME=/usr/lib/jvm/jdk1.8.0_131


--Ambari repository
sudo wget -nv http://public-repo-1.hortonworks.com/ambari/ubuntu14/2.x/updates/2.2.2.0/ambari.list -O /etc/apt/sources.list.d/ambari.list

sudo apt-get update

--Install ambari server only one one node
sudo apt-get install ambari-server

--Setup Ambari server
sudo ambari-server setup

sudo ambari-server start
sudo ambari-server status


http://ec2-10-10-10-10.compute-1.amazonaws.com:8080
admin/admin

--Verification

sudo -u hdfs hadoop fs -mkdir /user/ubuntu
sudo -u hdfs hadoop fs -chown ubuntu:ubuntu /user/ubuntu


ubuntu@ip-172-31-44-14:~$ hadoop fs -mkdir input
ubuntu@ip-172-31-44-14:~$ hadoop fs -copyFromLocal stocks input

ubuntu@ip-172-31-44-14:~$ hadoop jar MaxClosePrice-1.0.jar com.hirw.maxcloseprice.MaxClosePrice input output