# Exp 1 - HDFS commands
# local files used: emp.csv, notes.txt (keep them in the folder you run from)
# working dir in hdfs: /home/training

# ---- check the cluster is up ----
jps                                                    # should show NameNode, DataNode, ResourceManager...
hadoop version

# ---- directories ----
hadoop fs -mkdir -p /home/training/exp1                # -p makes parent dirs also
hadoop fs -ls /home/training                           # list
hadoop fs -ls -R /home/training                        # list recursively

# ---- local -> hdfs ----
hadoop fs -put emp.csv /home/training/exp1             # upload file
hadoop fs -copyFromLocal notes.txt /home/training/exp1 # same as put (only local src)
hadoop fs -ls /home/training/exp1

# ---- read ----
hadoop fs -cat /home/training/exp1/emp.csv             # print file
hadoop fs -tail /home/training/exp1/emp.csv            # last 1 KB
hadoop fs -text /home/training/exp1/notes.txt          # like cat, also reads compressed files

# ---- copy / move / rename inside hdfs ----
hadoop fs -mkdir /home/training/backup
hadoop fs -cp /home/training/exp1/emp.csv /home/training/backup/emp_copy.csv
hadoop fs -mv /home/training/backup/emp_copy.csv /home/training/backup/emp_old.csv   # mv = rename
hadoop fs -ls /home/training/backup

# ---- hdfs -> local ----
hadoop fs -get /home/training/exp1/emp.csv emp_from_hdfs.csv      # download
hadoop fs -copyToLocal /home/training/exp1/notes.txt notes_back.txt
hadoop fs -getmerge /home/training/exp1 merged.txt                # all files of dir -> 1 local file
ls -l emp_from_hdfs.csv notes_back.txt merged.txt

# ---- create / append ----
hadoop fs -touchz /home/training/exp1/empty.txt                   # 0 byte file
hadoop fs -appendToFile notes.txt /home/training/exp1/empty.txt   # add local file content at end
hadoop fs -cat /home/training/exp1/empty.txt

# ---- space, count, info ----
hadoop fs -du -h /home/training                                   # size of each item
hadoop fs -du -s -h /home/training                                # total size
hadoop fs -df -h /                                                # capacity of whole fs
hadoop fs -count /home/training                                   # dirs, files, bytes, path
hadoop fs -stat "%n %b bytes, repl %r, block %o" /home/training/exp1/emp.csv
hadoop fs -test -e /home/training/exp1/emp.csv && echo "file exists"

# ---- replication, permissions ----
hadoop fs -setrep 2 /home/training/exp1/emp.csv                   # change replication (no -w on 1-node vm, it waits forever)
hadoop fs -chmod 744 /home/training/exp1/emp.csv
hadoop fs -chown training:training /home/training/exp1/emp.csv   # needs superuser, may be denied
hadoop fs -ls /home/training/exp1                                 # 2nd column = replication

# ---- delete ----
hadoop fs -rm /home/training/backup/emp_old.csv
hadoop fs -rm -r /home/training/backup                            # dir: -r ; add -skipTrash to skip trash
hadoop fs -ls /home/training

# ---- admin (no "hadoop fs" form for these, they are hdfs tools) ----
hdfs fsck /home/training/exp1 -files -blocks -locations           # block health + where blocks are
hdfs dfsadmin -report                                             # datanodes, capacity
hdfs dfsadmin -safemode get                                       # safe mode on/off

# hadoop fs -help ls     -> help for any command

# ---- quick 3-command sets (for the HDFS part of R questions) ----
# 1) hadoop fs -mkdir /home/training/exam ; hadoop fs -put emp.csv /home/training/exam ; hadoop fs -ls /home/training/exam
# 2) hadoop fs -cat /home/training/exam/emp.csv ; hadoop fs -get /home/training/exam/emp.csv copy.csv ; hadoop fs -cp /home/training/exam/emp.csv /home/training/exam/emp2.csv
# 3) hadoop fs -mv /home/training/exam/emp2.csv /home/training/exam/old.csv ; hadoop fs -du -h /home/training/exam ; hadoop fs -rm /home/training/exam/old.csv
# 4) hadoop fs -setrep 2 /home/training/exam/emp.csv ; hadoop fs -stat %r /home/training/exam/emp.csv ; hadoop fs -chmod 644 /home/training/exam/emp.csv
# 5) hadoop fs -touchz /home/training/exam/a.txt ; hadoop fs -appendToFile notes.txt /home/training/exam/a.txt ; hadoop fs -tail /home/training/exam/a.txt
# 6) hadoop fs -count /home/training/exam ; hadoop fs -df -h / ; hdfs fsck /home/training/exam -files -blocks

# ---- viva ----
# namenode = metadata (file names, dir tree, block locations). datanode = actual blocks, heartbeat every 3 s
# secondary namenode only does checkpoint (merges edit log into fsimage), it is NOT a backup
# block size 128 MB default, replication 3 default. 300 MB file -> 128 + 128 + 44 MB blocks
# write once read many, good for big files, bad for lots of small files (namenode memory)
# hadoop fs works on any fs hadoop supports, hdfs dfs only on hdfs. same result on hdfs
# safe mode = read only state at startup till enough block reports come in
