#!/bin/bash
#
# usage: monitoring.bash ${EXP} ${YEAR} ${data} ${monitoringdir_local} ${monitoringhost_remote} ${monitoringdir_remote}

set -ex

site=daint

case ${site} in
   daint)  proxy="ssh daint";;
   ubelix) proxy="";;
   *)      proxy="";;
esac

module use /users/rhand/modules 
# module load cdo/1.9.8-gnu-6.0.5

scriptdir=${scriptdir:-/scratch/snx3000/rhand/experiments/PALAEO-RA/HIPPO_r001i1400p1f1801}

source ${scriptdir}/safe_copy.bash
source ${scriptdir}/plot_monitoring.bash

EXP=${1}
YEAR=${2}
data=${3}
monitoringdir_local=${4}
monitoringhost_remote=${5}
monitoringdir_remote=${6}
datahost_remote=${7}
meandir_remote=${8}

echo "executing monitoring command:"
echo "monitoring.bash $*"

cdo="cdo"
seaslist="djf jja"

# Show time stamp
echo "`date`: running monitoring procedure on output files"

# # Change to output directory
# cd $data

# creating timeseries plots

if [ ! -e $monitoringdir_local/timeseries ]; then mkdir -p $monitoringdir_local/timeseries ; fi
cd $monitoringdir_local/timeseries

if [ -e years_remote.dat ]; then rm years_remote.dat ; fi

if $(${proxy} ssh ${datahost_remote} ls ${meandir_remote}/${EXP}_????_mon.special.nc &>/dev/null); then
   filelist_remote=$(${proxy} ssh ${datahost_remote} ls ${meandir_remote}/${EXP}_????_mon.special.nc)
   echo ${filelist_remote} | sed -e "s&${meandir_remote}/${EXP}_&&g" -e "s/_mon.special.nc//g" | tr ' ' "\n" > years_remote.dat
else
   echo "" > years_remote.dat
fi

for var in rad0gm tsurfgm totprecgm; do

   case ${var} in
      totprecgm) mulc="-mulc,86400";;
      *)         mulc="";;
   esac

   cdo -selvar,${var} ${data}/${EXP}_${YEAR}_mon.special.nc ${var}.${YEAR}.nc
   
   if [ -e years_local.dat ]; then rm years_local.dat ; fi

   if $(ls ${var}.????.nc &>/dev/null); then
      ls ${var}.????.nc | sed -e "s/${var}.//" -e "s/.nc//" > years_local.dat
   else
      echo "" > years_local.dat
   fi

   pwd
   missing_years=$(comm -23 years_remote.dat years_local.dat)

   for miss_year in ${missing_years}; do 
      if [ ! -e ${EXP}_${miss_year}_mon.special.nc ] && $(${proxy} ssh ${datahost_remote} ls ${meandir_remote}/${EXP}_${miss_year}_mon.special.nc &>/dev/null) ; then 
         ypoc_efas ${datahost_remote} ${meandir_remote}/${EXP}_${miss_year}_mon.special.nc .
      fi
      cdo -selvar,${var} ${EXP}_${miss_year}_mon.special.nc ${var}.${miss_year}.nc
   done # miss_year

   cdo -r -O ${mulc} -mergetime ${var}.????.nc timeseries_${var}.nc
   plot_ts timeseries_${var}.nc timeseries_${YEAR}_${var}.png
   # cdo graph,device=png ${var}.all.mon.nc timeseries_${YEAR}_${var}
   safe_copy timeseries_${YEAR}_${var}.png ${monitoringhost_remote} ${monitoringdir_remote}/timeseries
   ${proxy} ssh ${monitoringhost_remote} mv ${monitoringdir_remote}/timeseries/timeseries_${YEAR}_${var}.png ${monitoringdir_remote}/timeseries/timeseries_${var}.png
   ${proxy} ssh ${monitoringhost_remote} chmod o+r ${monitoringdir_remote}/timeseries/timeseries_${var}.png

   cdo -r -O ${mulc} -yearmean -mergetime ${var}.????.nc timeseries_annual_${var}.nc
   plot_ts timeseries_annual_${var}.nc timeseries_annual_${YEAR}_${var}.png
   # cdo graph,device=png -yearmean -mergetime ${var}.????.nc timeseries_annual_${YEAR}_${var}
   safe_copy timeseries_annual_${YEAR}_${var}.png ${monitoringhost_remote} ${monitoringdir_remote}/timeseries/
   ${proxy} ssh ${monitoringhost_remote} mv ${monitoringdir_remote}/timeseries/timeseries_annual_${YEAR}_${var}.png ${monitoringdir_remote}/timeseries/timeseries_annual_${var}.png
   ${proxy} ssh ${monitoringhost_remote} chmod o+r ${monitoringdir_remote}/timeseries/timeseries_annual_${var}.png

done # var

# rm ${EXP}_????_mon.special.nc

# creating 2D plots

if [ ! -e $monitoringdir_local/2D ]; then mkdir -p $monitoringdir_local/2D ; fi
cd $monitoringdir_local/2D


function create_2d_plots {

   var=${1}

   case ${var} in
      totprec|temp2|tsurf|wind10|sn|aclcov|albedo)
               indata="-selvar,${var} ${data}/${EXP}_${YEAR}_mon.toasurf.nc"
      ;;
      wind[0-9]*00)
               level=${var/wind/}
               echo ${level}
               indata="-setvar,wind${level} -sellevel,${level}00 -expr,wind=sqrt(u*u+v*v) -sellevel,${level}00 ${data}/${EXP}_${YEAR}_mon.stdlevs.nc"
      ;;
   esac 

   case ${var} in
      totprec)  
                addc="" 
                mulc="-mulc,86400"
                ;;
      temp2|tsurf)
                addc="-addc,-273.15"
                mulc=""
                ;;
      wind10)
                addc=""
                mulc=""
                ;;
      *)        addc=""
                mulc=""
                ;;
   esac

   tmp=$(mktemp -d tmp_XXX)
   cd ${tmp}

   cdo -f nc copy ${indata} ${EXP}_${YEAR}_${var}.nc
   
   if [ -e ../${EXP}_allyears_${var}.nc -a "$(cdo showyear ../${EXP}_allyears_${var}.nc | sed -e "s/${YEAR}//g" -e "s/ //g")" != "" ]; then
      cdo delete,year=${YEAR} ../${EXP}_allyears_${var}.nc ${EXP}_old_${var}.nc
      cdo -O cat [ ${EXP}_{old,${YEAR}}_${var}.nc ] ${EXP}_allyears_${var}.nc && rm -f ${EXP}_{old,${YEAR}}_${var}.nc
   else
      mv ${EXP}_${YEAR}_${var}.nc ${EXP}_allyears_${var}.nc
   fi

   cdo --reduce_dim -r -timmean ${addc} ${mulc} -selyear,${YEAR} ${EXP}_allyears_${var}.nc tmp.year.${var}.nc &
   cdo --reduce_dim -r -timmean ${addc} ${mulc} ${EXP}_allyears_${var}.nc tmp.timmean.${var}.nc &
   wait

   plot_2d tmp.year.${var}.nc ${YEAR}_${var}.png &
   plot_2d tmp.timmean.${var}.nc timmean_${var}.png &
   wait

#   cdo grfill,device=png${clevs}${min}${max}${colour} -timmean ${addc} ${mulc} -selyear,${YEAR} ${EXP}_allyears_${var}.nc ${YEAR}
#   cdo grfill,device=png${clevs}${min}${max}${colour} -timmean ${addc} ${mulc} ${EXP}_allyears_${var}.nc timmean

   safe_copy timmean_${var}.png ${monitoringhost_remote} ${monitoringdir_remote}/2D/mean
   ${proxy} ssh ${monitoringhost_remote} chmod o+r ${monitoringdir_remote}/2D/mean/timmean_${var}.png
   safe_copy ${YEAR}_${var}.png ${monitoringhost_remote} ${monitoringdir_remote}/2D/${YEAR}
   ${proxy} ssh ${monitoringhost_remote} chmod o+r ${monitoringdir_remote}/2D/${YEAR}/${YEAR}_${var}.png

   for seas in ${seaslist} ; do

      cdo --reduce_dim -r -timmean ${addc} ${mulc} -selseas,${seas} -selyear,${YEAR} ${EXP}_allyears_${var}.nc tmp.year.${seas}.${var}.nc &
      cdo --reduce_dim -r -timmean ${addc} ${mulc} -selseas,${seas} ${EXP}_allyears_${var}.nc tmp.${seas}mean.${var}.nc &
      wait

      plot_2d tmp.${seas}mean.${var}.nc ${seas}mean_${var}.png &
      plot_2d tmp.year.${seas}.${var}.nc ${YEAR}_${seas}_${var}.png &
      wait

      # cdo grfill,device=png$f{clevs}${min}${max}${colour} -timmean ${addc} ${mulc} -selseas,${seas} -selyear,${YEAR} ${EXP}_allyears_${var}.nc ${YEAR}_${seas}
      # cdo grfill,device=png${clevs}${min}${max}${colour} -timmean ${addc} ${mulc} -selseas,${seas} ${EXP}_allyears_${var}.nc ${seas}mean

      safe_copy ${seas}mean_${var}.png ${monitoringhost_remote} ${monitoringdir_remote}/2D/mean
      ${proxy} ssh ${monitoringhost_remote} chmod o+r ${monitoringdir_remote}/2D/mean/${seas}mean_${var}.png
      safe_copy ${YEAR}_${seas}_${var}.png ${monitoringhost_remote} ${monitoringdir_remote}/2D/${YEAR}
      ${proxy} ssh ${monitoringhost_remote} chmod o+r ${monitoringdir_remote}/2D/${YEAR}/${YEAR}_${seas}_${var}.png

   done # seas

   mv ${EXP}_allyears_${var}.nc ..

   cd ..

   rm -rf ${tmp}

}


for var in totprec temp2 tsurf wind10 wind500 sn aclcov albedo ; do

   create_2d_plots ${var} &

done # var

wait

chmod -R g+w $monitoringdir_local

# ${scriptdir}/collect_jobinfo.bash

exit 0
