Saturday, March 2, 2019

Haddop Working with HDFS

### LOCAL FILE SYSTEM ###

ls
mkdir
cp
mv
rm

### LISTING ROOT DIRECTORY ###

hadoop fs -ls /

### LISTING DEFAULT TO HOME DIRECTORY ###

hadoop fs -ls

hadoop fs -ls /user/deepakdubey

### CREATE A DIRECTORY IN HDFS ###

hadoop fs -mkdir hadoop-file-system-test1

### COPY FROM LOCAL FS TO HDFS ###

hadoop fs -copyFromLocal  /deepakdubey-starterkit/hdfs/commands/stocks-exchange.cvs hadoop-file-system-test1


### COPY TO HDFS TO LOCAL FS ###

hadoop fs -copyToLocal hadoop-file-system-test1/stocks-exchange.cvs .

hadoop fs -ls hadoop-file-system-test1

### CREATE 2 MORE DIRECTORIES ###

hadoop fs -mkdir hadoop-file-system-test2
hadoop fs -mkdir hadoop-file-system-test3

### COPY A FILE FROM ONE FOLDER TO ANOTHER ###

hadoop fs -cp hadoop-file-system-test1/stocks-exchange.cvs hadoop-file-system-test2

### MOVE A FILE FROM ONE FOLDER TO ANOTHER ###

hadoop fs -mv hadoop-file-system-test1/stocks-exchange.cvs hadoop-file-system-test3

### CHECK REPLICATION ###

hadoop fs -ls hadoop-file-system-test3

### CHANGE OR SET REPLICATION FACTOR ###

hadoop fs -Ddfs.replication=2 -cp hadoop-file-system-test2/stocks-exchange.cvs hadoop-file-system-test2/test-with-replication-factor-2.csv

hadoop fs -ls hadoop-file-system-test2

hadoop fs -ls hadoop-file-system-test2/test-with-replication-factor-2.csv

### CHANGING PERMISSIONS ###

hadoop fs -chmod 777 hadoop-file-system-test2/test-with-replication-factor-2.csv

### FILE SYSTEM CHECK - REQUIRES ADMIN PREVILEGES ###

sudo -u hdfs hdfs fsck /user/deepakdubey/hadoop-file-system-test2 -files -blocks -locations

sudo -u hdfs hdfs fsck /user/deepakdubey/hadoop-file-system-test3 -files -blocks -locations

sudo -u hdfs hdfs fsck /user/ubuntu/input/yelp/academic_dataset_review.json -files -blocks -locations

vi /etc/hadoop/conf/hdfs-site.xml

/data/1/dfs/dn/current/BP-2125152513-172.31.45.216-1410037307133/current/finalized


### DELETE DIR/FILES IN HDFS ###


hadoop fs -rm hadoop-file-system-test2/test-with-replication-factor-5.csv


hadoop fs -rm -r hadoop-file-system-test1

hadoop fs -rm -r hadoop-file-system-test2

hadoop fs -rm -r hadoop-file-system-test3