2019-05-14

function texec {
    echo "$@"
    "$@"
    local status=$?
    if [ $status -ne 0 ]; then
        echo "error with $1"
        exit $status
    fi
    return $status
}

function iis {
        folder=$1
        iis_folder=${folder}_iis

        for file in $TMPFOLDER/${folder}/part*; do
                filename="${file##*/}"
                texec time $bin/run.sh com.ebay.gro.iis.SMagicIISData $file $TMPFOLDER/$iis_folder/$filename.txt
                if [ -s "$TMPFOLDER/${iis_folder}/$filename.txt" ]
                then
                        echo "$TMPFOLDER/${iis_folder}/$filename.txt done!"
                else
                        echo "$TMPFOLDER/${iis_folder}/$filename.txt zero size, deleted!"
                        rm $TMPFOLDER/${iis_folder}/$filename.txt
                fi
        done
}

# Step 1: Clean Tmp
rm -rf $TMPFOLDER/${SITE}
mkdir $TMPFOLDER/${SITE}

# Step 2: Download Results to Local FS
texec /apache/hadoop/hadoop/bin/hadoop fs -copyToLocal $HDFSFOLDER/part* $TMPFOLDER/${SITE}
#hadoop fs -rm -r -skipTrash $HDFSFOLDER
# Step 3: Call IIS Service & Dump Items which have comp price and deals
rm -rf $TMPFOLDER/${SITE}_iis
mkdir $TMPFOLDER/${SITE}_iis

iis ${SITE}

# Step 4: Upload Results to HDFS folder
/apache/hadoop/hadoop/bin/hadoop fs -rm -r -skipTrash $PIGFOLDER/${SITE}
/apache/hadoop/hadoop/bin/hadoop fs -mkdir $PIGFOLDER/${SITE}

texec /apache/hadoop/hadoop/bin/hadoop fs -copyFromLocal $TMPFOLDER/${SITE}_iis $PIGFOLDER/${SITE}