# Exp 4 - Pig
# files: emp.csv (id,name,dept,salary), dept.csv (dept,city), words.txt, script.pig

# ---- put input in hdfs ----
hadoop fs -mkdir -p /home/training/pig
hadoop fs -put emp.csv dept.csv words.txt /home/training/pig/
hadoop fs -rm -r -f /home/training/pig_out          # STORE fails if output dir already exists

# ---- run ----
pig script.pig                 # mapreduce mode (reads hdfs), whole script
pig                            # grunt> shell, type statements one by one
pig -x local                   # local mode, reads files from local disk (use LOAD 'emp.csv')

# ---- output ----
hadoop fs -ls /home/training/pig_out
hadoop fs -cat /home/training/pig_out/part*

# ---- inside grunt, useful ----
# DESCRIBE emp;      schema
# EXPLAIN stats;     logical / physical / mapreduce plan
# ILLUSTRATE stats;  sample rows through each step
# fs -ls /home/training/pig     hdfs commands from grunt
# quit

# ---- expected ----
# high  -> (2,Priya,IT,72000) (4,Sneha,IT,61000) (5,Rohit,Finance,54000)
# stats -> (HR,2,47000.0,49000) (IT,2,66500.0,72000) (Sales,1,38000.0,38000) (Finance,1,54000.0,54000)
# word count -> (big,3) (data,3) ...

# ---- viva ----
# pig latin = procedural data flow language, pig turns it into mapreduce jobs (less code than java)
# modes: local (-x local) and mapreduce (default, hdfs)
# types: int long float double chararray bytearray boolean ; complex: tuple (a,b), bag {(a),(b)}, map [k#v]
# relation = bag of tuples. lazy: nothing runs till DUMP or STORE
# GROUP = one relation, COGROUP = 2 or more relations by same key
# DUMP prints on screen, STORE writes to file system
# FLATTEN un-nests a bag/tuple (used after TOKENIZE)
# pig vs hive: pig procedural, schema optional, ETL ; hive declarative sql, needs schema, reports
