Hadoop HDFS Admin - Network Topology Of Cluster

sudo vi /etc/hadoop/conf/core-site.xml


<property>

    <name>net.topology.node.switch.mapping.impl</name>

    <value>org.apache.hadoop.net.ScriptBasedMapping</value>

</property>



<property>

    <name>net.topology.script.file.name</name>

    <value>/etc/hadoop/conf/topology.sh</value>

</property>



sudo vi /etc/hadoop/conf/topology.data



10-10-10-10 /rack1

20-20-20-20 /rack1

30-30-30-30 /rack1

40-40-40-40 /rack2



--Topology script will be called when Namenode starts


./etc/hadoop/conf/topology.sh 10-10-10-10 20-20-20-20 30-30-30-30 40-40-40-40


sudo vi /etc/hadoop/conf/topology.sh



#!/bin/bash

HADOOP_CONF=/etc/hadoop/conf

while [ $# -gt 0 ] ; do

  nodeArg=$1

  exec< ${HADOOP_CONF}/topology.data

  result=""

  while read line ; do

    ar=( $line )

    if [ "${ar[0]}" = "$nodeArg" ] ; then

      result="${ar[1]}"

    fi

  done

  shift

  if [ -z "$result" ] ; then

     echo -n "/rack0"

  else

    echo -n "$result "

  fi

done



--IP Address encoded with rack information


xx.xx.rackno.nodeno

50-50-50-50

Day to Day Esentails - Adding and Removing Nodes

--Marking Datanode Dead

2 * dfs.namenode.heartbeat.recheck-interval + 10 * dfs.heartbeat.interval

dfs.heartbeat.interval - defaults to 3 seconds

dfs.namenode.heartbeat.recheck-interval - defaults to 300,000 milliseconds = 300 seconds

2 * 300 seconds + 10 * 3 seconds = 630 seconds (10 minutes and 30 seconds)

--To commission or decommission datanodes

sudo vi /etc/hadoop/conf/hdfs-site.xml

<property>

  <name>dfs.hosts</name>

  <value>/etc/hadoop/conf/include</value>

</property>


<property>

  <name>dfs.hosts.exclude</name>

  <value>/etc/hadoop/conf/exclude</value>

</property>


sudo vi /etc/hadoop/conf/include

sudo vi /etc/hadoop/conf/exclude

--After include/exclude changes, refresh nodes for changes to take effect

hdfs dfsadmin -refreshNodes



http://ec2-10-10-10-10.compute-1.amazonaws.com:50070/dfshealth.html

--To commission or decommission nodemanagers

sudo vi /etc/hadoop/conf/yarn-site.xml

<property>

  <name>yarn.resourcemanager.nodes.include-path</name>

  <value>/etc/hadoop/conf/include</value>

</property>


<property>

  <name>yarn.resourcemanager.nodes.exclude-path</name>

  <value>/etc/hadoop/conf/exclude</value>

</property>



--After include/exclude changes, refresh nodes for changes to take effect



yarn rmadmin -refreshNodes

--Support graceful decommission of nodemanager


https://issues.apache.org/jira/browse/YARN-914

Puppet Installation

--Public DNS
ec2-10-10-10-10.compute-1.amazonaws.com
ec2-20-20-20-20.compute-1.amazonaws.com
ec2-30-30-30-30.compute-1.amazonaws.com
ec2-40-40-40-40.compute-1.amazonaws.com

--Private DNS
ip-50-50-50-50.ec2.internal
ip-60-60-60-60.ec2.internal
ip-70-70-70-70.ec2.internal
ip-80-80-80-80.ec2.internal

--Download Puppet on all nodes
wget https://apt.puppetlabs.com/puppet5-release-xenial.deb
sudo dpkg -i puppet5-release-xenial.deb
sudo apt update

--Install puppetserver on Puppet Master

sudo apt-get install puppetserver

--Install agents on 3 nodes

sudo apt-get install puppet-agent

--Update Puppet configuration

sudo vi /etc/puppetlabs/puppet/puppet.conf

server=ip-50-50-50-50.ec2.internal

runinterval = 1800

--Start Puppet server
sudo service puppetserver start

--Start Puppet agent
sudo /opt/puppetlabs/bin/puppet resource service puppet ensure=running enable=true

--Sign Certificates

sudo /opt/puppetlabs/bin/puppet cert list

sudo /opt/puppetlabs/bin/puppet cert sign ip-60-60-60-60.ec2.internal
sudo /opt/puppetlabs/bin/puppet cert sign ip-70-70-70-70.ec2.internal
sudo /opt/puppetlabs/bin/puppet cert sign ip-80-80-80-80.ec2.internal

Puppet Concepts with Tomcat Installation

--Private DNS
ip-10-10-10-10.ec2.internal (Master)
ip-20-20-20-20.ec2.internal (Agent 1)
ip-30-30-30-30.ec2.internal (Agent 2)
ip-40-40-40-40.ec2.internal (Agent 3)

--Create module tomcat
cd /etc/puppetlabs/code/environments/production/modules

mkdir tomcat

--Create manifest folder

cd tomcat

mkdir manifests

--Create init.pp (class name should match the name of the module)

vi init.pp

class tomcat {
}

--Create install.pp

vi install.pp

class tomcat::install {

 package{'tomcat8':
  ensure => installed
 } 
 
 package{'tomcat8-admin':
  ensure => installed
 }
 
}

--Create start.pp

vi start.pp

class tomcat::start{

 service{'tomcat8' :
  ensure => running
 }
 
}

--Node assignments

cd /etc/puppetlabs/code/environments/production/manifests

vi tomcat.pp

node 'ip-20-20-20-20.ec2.internal' {
 include tomcat::install
 include tomcat::start
}

--To run agent "pull" on as needed basis

sudo /opt/puppetlabs/bin/puppet agent --test

--Create config.pp

cd /etc/puppetlabs/code/environments/production/modules/tomcat/manifests

vi config.pp

class tomcat::config {
 
 file { '/etc/tomcat8/tomcat-users.xml':
  source   => 'puppet:///modules/tomcat/tomcat-users.xml',
  owner    => 'tomcat8', 
  group    => 'tomcat8', 
  mode     => '0600',
  notify   => Service['tomcat8'] 
 }
 
}


--Copy tomcat-users.xml file


--Add config to tomcat.pp

cd /etc/puppetlabs/code/environments/production/manifests

vi tomcat.pp

node 'ip-20-20-20-20.ec2.internal' {
 include tomcat::install
 include tomcat::config
 include tomcat::start
}

Haddop Working with HDFS

### LOCAL FILE SYSTEM ###

ls
mkdir
cp
mv
rm

### LISTING ROOT DIRECTORY ###

hadoop fs -ls /

### LISTING DEFAULT TO HOME DIRECTORY ###

hadoop fs -ls

hadoop fs -ls /user/deepakdubey

### CREATE A DIRECTORY IN HDFS ###

hadoop fs -mkdir hadoop-file-system-test1

### COPY FROM LOCAL FS TO HDFS ###

hadoop fs -copyFromLocal  /deepakdubey-starterkit/hdfs/commands/stocks-exchange.cvs hadoop-file-system-test1


### COPY TO HDFS TO LOCAL FS ###

hadoop fs -copyToLocal hadoop-file-system-test1/stocks-exchange.cvs .

hadoop fs -ls hadoop-file-system-test1

### CREATE 2 MORE DIRECTORIES ###

hadoop fs -mkdir hadoop-file-system-test2
hadoop fs -mkdir hadoop-file-system-test3

### COPY A FILE FROM ONE FOLDER TO ANOTHER ###

hadoop fs -cp hadoop-file-system-test1/stocks-exchange.cvs hadoop-file-system-test2

### MOVE A FILE FROM ONE FOLDER TO ANOTHER ###

hadoop fs -mv hadoop-file-system-test1/stocks-exchange.cvs hadoop-file-system-test3

### CHECK REPLICATION ###

hadoop fs -ls hadoop-file-system-test3

### CHANGE OR SET REPLICATION FACTOR ###

hadoop fs -Ddfs.replication=2 -cp hadoop-file-system-test2/stocks-exchange.cvs hadoop-file-system-test2/test-with-replication-factor-2.csv

hadoop fs -ls hadoop-file-system-test2

hadoop fs -ls hadoop-file-system-test2/test-with-replication-factor-2.csv

### CHANGING PERMISSIONS ###

hadoop fs -chmod 777 hadoop-file-system-test2/test-with-replication-factor-2.csv

### FILE SYSTEM CHECK - REQUIRES ADMIN PREVILEGES ###

sudo -u hdfs hdfs fsck /user/deepakdubey/hadoop-file-system-test2 -files -blocks -locations

sudo -u hdfs hdfs fsck /user/deepakdubey/hadoop-file-system-test3 -files -blocks -locations

sudo -u hdfs hdfs fsck /user/ubuntu/input/yelp/academic_dataset_review.json -files -blocks -locations

vi /etc/hadoop/conf/hdfs-site.xml

/data/1/dfs/dn/current/BP-2125152513-172.31.45.216-1410037307133/current/finalized


### DELETE DIR/FILES IN HDFS ###


hadoop fs -rm hadoop-file-system-test2/test-with-replication-factor-5.csv


hadoop fs -rm -r hadoop-file-system-test1

hadoop fs -rm -r hadoop-file-system-test2

hadoop fs -rm -r hadoop-file-system-test3