#!/bin/ksh

#Script: sw_dvs_not_on_web
#        This is the official OSW_Scripts version.
#        Please direct questions, comments, etc... to GS-W_OSW_Scripts@usgs.gov

#Usage: sw_dvs_not_on_web

#Purpose: Summarize DV's stored in primary discharge, sediment, and precipitation DD's in database that aren't currently
#         being published on NWISWeb.

#Original Author: Wade Walker, CO WSC / OSW

#
#History:
#
#  3/16/2010 - initial coding of sed_dvs_not_on_web -WJW
#  2/13/2012 - expanded to discharge and precipitation DD's -WJW
#  8/15/2012 - Updated for NWIS 5.0/Oracle -WJW
#  1/31/2013 - Swapped metric discharge parameter codes. Dropped times from dvinv dates. Readied for official release.-WJW
#
version=5.0

# Source the NWIS environment
. /usr/opt/etc/nwis.profile
. $NWISHOME/util/nwo_set_nwis_env

db_no='01'
agency=USGS

# function to get and verify a table name from the database
get_table_name()
{
  # Use tsql to get the table name suffix from the database and construct
  # the actual table name
  #
  # Use like this:
  #
  # get_table_name dd
  # ddtbl=${table_name}
  #
  lcDDLnm=`echo "${1}" | tr '[A-Z]' '[a-z]'`
  ucDDLnm=`echo "${1}" | tr '[a-z]' '[A-Z]'`
  ret_table_name=${lcDDLnm}`/usr/local/bin/tsql ${NWISDB} "select suffix_tx \
        from ${NWIS_SCHEMA}.master_table where ddl_nm='${ucDDLnm}_##' AND db_no='${db_no}'"`
  # Verify we actually got a table name
  if [ -z "${ret_table_name}" ] ; then
    echo
    echo **** Fatal error - no table name retrieved for ddl_nm ${ucDDLnm}'_##' database ${db_no}
    echo
    exit 7
  fi
  table_name="${NWIS_SCHEMA}.${ret_table_name}"
}

get_table_name sitefile
sitetbl=${table_name}
get_table_name loc
loctbl=${table_name}
get_table_name dd
ddtbl=${table_name}
get_table_name dv
dvtbl=${table_name}
get_table_name dv_data
dvdtbl=${table_name}


# define cleanup function
function cleanup
{
 rm -f db.tmp wdvsl*.rdb wdvsc*.rdb wdvs.rdb wdvsq*.rdb wdvsp*.rdb tmp.seddvs.out mdvsc mdvsl mdvsp mdvsq
}
clear

# set traps to cleanup temp files
trap '
print "Do you really want to quit sw_dvs_not_on_web and clean up files? (CR=y) \c"
read quitans
case $quitans in
 n | N | no | NO | No | nO)
  quitans=
  print "Continuing";;
 *)
  print "Cleaning up files...exiting sw_dvs_not_on_web"
   cleanup
   print "Bye"
   exit
    ;;
esac
  ' INT

trap '
  print "Cleaning up files...exiting sw_dvs_not_on_web"
   cleanup
   print "Bye"
   exit
  ' QUIT

trap '
   print "Cleaning up files...exiting sw_dvs_not_on_web"
   cleanup
   print "Bye"
   exit
  ' EXIT

trap '
    print "Cleaning up files...exiting sw_dvs_not_on_web"
   cleanup
   print "Bye"
   exit
  ' HUP

trap '
   print "Cleaning up files...exiting sw_dvs_not_on_web"
   cleanup
   print "Bye"
   exit
  ' TERM



print
echo "Version $version"
echo
print "_____________________ sw_dvs_not_on_web ___________________________"
print "_                                                                 _"
print "_ Summarize DVs stored in primary SW related DDs in DB01 that     _"
print "_        are not currently being displayed on NWISWeb             _"
print "_                                                                 _"
print "___________________________________________________________________"
print 
print 
print "Attempting to default to current server"
print $NWISDB > db.tmp
sed 's/\/\@/ /g' <db.tmp | read nwdb server crap
rm -f db.tmp

case $server in
   nwis?? | NWIS?? )
     nwserver=$server
     ;;
   * )
until [ $nwserver ]
do
print "Unable to find current server name, enter NWIS server name you are running this on:\c"
read server

case $server in
   nwis?? | NWIS?? )
     nwserver=$server
     ;;
    q | qu | quit | ex | exit | Q | QU | QUIT | EX | EXIT )
     print "Exiting from script"
     exit
     ;;
    * )
     print "Entry not understood"
     print "Please enter just base server name."
     print "i.e. - nwisco or nwistx or nwispr"
     sleep 6
     nwserver=''
     ;;
esac
done
;;
esac
print ""
print "Using nwis server $nwserver"
print ""
print "Pulling list of SW related DVs currently displayed on NWISWeb for this server"
print ""
print "Search is currently restricted to these parameters:"
print "Sediment parameters 80154 and 80155"
print "Discharge parameters 00060 and 30208"
print "Precipitation parameter 00045"
print ""
print ""
lwp-request "http://nwis.waterdata.usgs.gov/nwisweb/local/nwis_host/$nwserver/local/support/$nwserver.dd_list.dv.01.rdb" >wdvs.rdb
row parameter_cd eq "80154" and access_cd eq "0" < wdvs.rdb > wdvsc.rdb
row parameter_cd eq "80155" and access_cd eq "0" < wdvs.rdb > wdvsl.rdb
row parameter_cd eq "00060" or parameter_cd eq "30208" and access_cd eq "0" < wdvs.rdb > wdvsq.rdb
row parameter_cd eq "00045" and access_cd eq "0" < wdvs.rdb > wdvsp.rdb
column agency_cd site_no dd_nu stat_cd < wdvsl.rdb > wdvsl2a.rdb
column agency_cd site_no dd_nu stat_cd < wdvsc.rdb > wdvsc2a.rdb
column agency_cd site_no dd_nu stat_cd < wdvsq.rdb >  wdvsq2a.rdb 
column agency_cd site_no dd_nu stat_cd < wdvsp.rdb >  wdvsp2a.rdb 
sed 's/agency_cd/Agncy/g' <wdvsl2a.rdb > wdvsl2.rdb
sed 's/agency_cd/Agncy/g' <wdvsc2a.rdb > wdvsc2.rdb
sed 's/agency_cd/Agncy/g' <wdvsq2a.rdb > wdvsq2.rdb
sed 's/agency_cd/Agncy/g' <wdvsp2a.rdb > wdvsp2.rdb

print "THE FOLLOWING DAILY SEDIMENT CONCENTRATION DATA ARE CURRENTLY BEING SERVED ON NWISWEB:" > tmp.seddvs.out
/usr/local/bin/ptbl -p0 -l132 < wdvsc2.rdb >> tmp.seddvs.out
print "" >> tmp.seddvs.out
print "" >> tmp.seddvs.out
print "THE FOLLOWING DAILY SEDIMENT LOAD DATA ARE CURRENTLY BEING SERVED ON NWISWEB:" >> tmp.seddvs.out
/usr/local/bin/ptbl -p0 -l132 < wdvsl2.rdb >> tmp.seddvs.out
print "" >> tmp.seddvs.out
print "" >> tmp.seddvs.out

print "THE FOLLOWING DAILY WATER DISCHARGE DATA ARE CURRENTLY BEING SERVED ON NWISWEB:" >> tmp.seddvs.out
/usr/local/bin/ptbl -p0 -l132 < wdvsq2.rdb >> tmp.seddvs.out
print "" >> tmp.seddvs.out
print "" >> tmp.seddvs.out

print "THE FOLLOWING DAILY PRECIPITATION DATA ARE CURRENTLY BEING SERVED ON NWISWEB:" >> tmp.seddvs.out
/usr/local/bin/ptbl -p0 -l132 < wdvsp2.rdb >> tmp.seddvs.out
print "" >> tmp.seddvs.out
print "" >> tmp.seddvs.out

/usr/local/bin/headchg -del < wdvsl2.rdb > wdvsl3.rdb
/usr/local/bin/headchg -del < wdvsc2.rdb > wdvsc3.rdb
/usr/local/bin/headchg -del < wdvsq2.rdb > wdvsq3.rdb
/usr/local/bin/headchg -del < wdvsp2.rdb > wdvsp3.rdb
print ""
print "Pulling inventory of all data on $nwserver"
print ""

print "Agency\tStation\tDD\tStat\tPrimary\tParm\tNdays\tFrom\tTo\tBegins\tEnds" >mdvsl
print "Agency\tStation\tDD\tStat\tPrimary\tParm\tNdays\tFrom\tTo\tBegins\tEnds" >mdvsc
print "Agency\tStation\tDD\tStat\tPrimary\tParm\tNdays\tFrom\tTo\tBegins\tEnds" >mdvsq
print "Agency\tStation\tDD\tStat\tPrimary\tParm\tNdays\tFrom\tTo\tBegins\tEnds" >mdvsp

print "6S\t15S\t4N\t5S\t7S\t5S\t8N\t12N\t12N\t12S\t12S" >> mdvsl
print "6S\t15S\t4N\t5S\t7S\t5S\t8N\t12N\t12N\t12S\t12S" >> mdvsc
print "6S\t15S\t4N\t5S\t7S\t5S\t8N\t12N\t12N\t12S\t12S" >> mdvsq
print "6S\t15S\t4N\t5S\t7S\t5S\t8N\t12N\t12N\t12S\t12S" >> mdvsp

# set date format to just date
dtFormatQuery="alter session set NLS_DATE_FORMAT='YYYY-MM-DD'"


   /usr/local/bin/tsql $NWISDB  "${dtFormatQuery}" "select s.agency_cd,s.site_no,dd.dd_nu,dv.stat_cd,dd.dd_official_fg,dd.parm_cd,count(dvd.dv_dt),min(dvd.dv_va),max(dvd.dv_va),min(dvd.dv_dt),max(dvd.dv_dt) from ${sitetbl} s, ${loctbl} l, ${ddtbl} dd, ${dvtbl} dv, ${dvdtbl} dvd where l.site_id=s.site_id and dd.loc_id=l.loc_id and dd.dd_official_fg='Y' and dd.parm_cd='80154' and dv.dd_id=dd.dd_id and dvd.dv_id=dv.dv_id group by s.agency_cd,s.site_no,dd.dd_nu,dv.stat_cd,dd.dd_official_fg,dd.parm_cd order by s.agency_cd,s.site_no,dd.dd_nu,dv.stat_cd,dd.parm_cd" >> mdvsc

   /usr/local/bin/tsql $NWISDB  "${dtFormatQuery}" "select s.agency_cd,s.site_no,dd.dd_nu,dv.stat_cd,dd.dd_official_fg,dd.parm_cd,count(dvd.dv_dt),min(dvd.dv_va),max(dvd.dv_va),min(dvd.dv_dt),max(dvd.dv_dt) from ${sitetbl} s, ${loctbl} l, ${ddtbl} dd, ${dvtbl} dv, ${dvdtbl} dvd where l.site_id=s.site_id and dd.loc_id=l.loc_id and dd.dd_official_fg='Y' and dd.parm_cd='80155' and dv.dd_id=dd.dd_id and dvd.dv_id=dv.dv_id group by s.agency_cd,s.site_no,dd.dd_nu,dv.stat_cd,dd.dd_official_fg,dd.parm_cd order by s.agency_cd,s.site_no,dd.dd_nu,dv.stat_cd,dd.parm_cd" >> mdvsl

   /usr/local/bin/tsql $NWISDB  "${dtFormatQuery}" "select s.agency_cd,s.site_no,dd.dd_nu,dv.stat_cd,dd.dd_official_fg,dd.parm_cd,count(dvd.dv_dt),min(dvd.dv_va),max(dvd.dv_va),min(dvd.dv_dt),max(dvd.dv_dt) from ${sitetbl} s, ${loctbl} l, ${ddtbl} dd, ${dvtbl} dv, ${dvdtbl} dvd where l.site_id=s.site_id and dd.loc_id=l.loc_id and dd.dd_official_fg='Y' and dd.parm_cd in ('00060', '99060') and dv.dd_id=dd.dd_id and dvd.dv_id=dv.dv_id group by s.agency_cd,s.site_no,dd.dd_nu,dv.stat_cd,dd.dd_official_fg,dd.parm_cd order by s.agency_cd,s.site_no,dd.dd_nu,dv.stat_cd,dd.parm_cd" >> mdvsq

   /usr/local/bin/tsql $NWISDB  "${dtFormatQuery}" "select s.agency_cd,s.site_no,dd.dd_nu,dv.stat_cd,dd.dd_official_fg,dd.parm_cd,count(dvd.dv_dt),min(dvd.dv_va),max(dvd.dv_va),min(dvd.dv_dt),max(dvd.dv_dt) from ${sitetbl} s, ${loctbl} l, ${ddtbl} dd, ${dvtbl} dv, ${dvdtbl} dvd where l.site_id=s.site_id and dd.loc_id=l.loc_id and dd.dd_official_fg='Y' and dd.parm_cd='00045' and dv.dd_id=dd.dd_id and dvd.dv_id=dv.dv_id group by s.agency_cd,s.site_no,dd.dd_nu,dv.stat_cd,dd.dd_official_fg,dd.parm_cd order by s.agency_cd,s.site_no,dd.dd_nu,dv.stat_cd,dd.parm_cd" >> mdvsp

#cp mdvsc mdvsc.keep
#cp mdvsl mdvsl.keep

#/usr/opt/osw/dvinv -p -m 80154 > dvinvc
#/usr/opt/osw/dvinv -p -m 80155 > dvinvl

print "THE FOLLOWING DAILY SEDIMENT CONCENTRATION DATA ARE STORED IN THE DATABASE:" >> tmp.seddvs.out
/usr/local/bin/ptbl -p0 -l132 < mdvsc >> tmp.seddvs.out
print "" >> tmp.seddvs.out
print "" >> tmp.seddvs.out
print "THE FOLLOWING DAILY SEDIMENT LOAD DATA ARE STORED IN THE DATABASE:" >> tmp.seddvs.out
/usr/local/bin/ptbl -p0 -l132 < mdvsl >> tmp.seddvs.out
print "" >> tmp.seddvs.out
print "" >> tmp.seddvs.out
print "THE FOLLOWING DAILY WATER DISCHARGE DATA ARE STORED IN THE DATABASE:" >> tmp.seddvs.out
/usr/local/bin/ptbl -p0 -l132 < mdvsq >> tmp.seddvs.out
print "" >> tmp.seddvs.out
print "" >> tmp.seddvs.out
print "THE FOLLOWING DAILY PRECIPITATION DATA ARE STORED IN THE DATABASE:" >> tmp.seddvs.out
/usr/local/bin/ptbl -p0 -l132 < mdvsq >> tmp.seddvs.out
print "" >> tmp.seddvs.out
print "" >> tmp.seddvs.out


#grep -v Agency < dvinvc > dvinvc1
#grep -v Agency < dvinvl > dvinvl1


#grep -v "\-\-\-\-\-\-" < dvinvc1 >> mdvsc
#grep -v "\-\-\-\-\-\-" < dvinvl1 >> mdvsl
print ""
print "Finding data stored in database in primary DDs not being served on NWISWeb"
print ""

while read agncy sta dd stat crap
do
case $dd in
 01 )
  dd="1"
  ;;
 02 )
  dd="2"
  ;;
 03 )
  dd="3"
  ;;
 04 )
  dd="4"
  ;;
 05 )
  dd="5"
  ;;
 06 )
  dd="6"
  ;;
 07 )
  dd="7"
  ;;
 08 )
  dd="8"
  ;;
 09 )
  dd="9"
  ;;
esac
#print "conc dd $dd"
grep -v "$agncy	$sta	$dd	$stat" <mdvsc> missdvc
mv missdvc mdvsc
done < wdvsc3.rdb

while read agncy sta dd stat crap 
do
case $dd in
 01 )
  dd="1"
  ;;
 02 )
  dd="2"
  ;;
 03 )
  dd="3"
  ;;
 04 )
  dd="4"
  ;;
 05 )
  dd="5"
  ;;
 06 )
  dd="6"
  ;;
 07 )
  dd="7"
  ;;
 08 )
  dd="8"
  ;;
 09 )
  dd="9"
  ;;
esac
#print "load dd $dd"
grep -v "$agncy	$sta	$dd	$stat" <mdvsl> missdvl
#/usr/local/bin/row Station ne "$sta" and DD ne "$dd" and Stat ne "$stat" <mdvsl> missdvl

mv missdvl mdvsl
done < wdvsl3.rdb

while read agncy sta dd stat crap 
do
case $dd in
 01 )
  dd="1"
  ;;
 02 )
  dd="2"
  ;;
 03 )
  dd="3"
  ;;
 04 )
  dd="4"
  ;;
 05 )
  dd="5"
  ;;
 06 )
  dd="6"
  ;;
 07 )
  dd="7"
  ;;
 08 )
  dd="8"
  ;;
 09 )
  dd="9"
  ;;
esac
#print "load dd $dd"
grep -v "$agncy	$sta	$dd	$stat" <mdvsq> missdvq
#/usr/local/bin/row Station ne "$sta" and DD ne "$dd" and Stat ne "$stat" <mdvsl> missdvl

mv missdvq mdvsq
done < wdvsq3.rdb

while read agncy sta dd stat crap 
do
case $dd in
 01 )
  dd="1"
  ;;
 02 )
  dd="2"
  ;;
 03 )
  dd="3"
  ;;
 04 )
  dd="4"
  ;;
 05 )
  dd="5"
  ;;
 06 )
  dd="6"
  ;;
 07 )
  dd="7"
  ;;
 08 )
  dd="8"
  ;;
 09 )
  dd="9"
  ;;
esac
#print "load dd $dd"
grep -v "$agncy	$sta	$dd	$stat" <mdvsp> missdvp
#/usr/local/bin/row Station ne "$sta" and DD ne "$dd" and Stat ne "$stat" <mdvsl> missdvl

mv missdvp mdvsp
done < wdvsp3.rdb


/usr/bin/date | read date

print "sw_dvs_not_on_web output produced $date by $USER" > "$nwserver"_sw_dvs_not_on_web.txt
print "" >> "$nwserver"_sw_dvs_not_on_web.txt
print "Original filename = "$nwserver"_sw_dvs_not_on_web.txt" >> "$nwserver"_sw_dvs_not_on_web.txt
print "" >> "$nwserver"_sw_dvs_not_on_web.txt
print "" >> "$nwserver"_sw_dvs_not_on_web.txt
print "DVs found to be missing from NWISWeb should be verified as good, quality controlled data and added" >> "$nwserver"_sw_dvs_not_on_web.txt
print " to NWISWeb as appropriate." >> "$nwserver"_sw_dvs_not_on_web.txt
print "" >> "$nwserver"_sw_dvs_not_on_web.txt
print "If data are not quality controlled and of little value, they should be deleted from the DD." >> "$nwserver"_sw_dvs_not_on_web.txt
print "If data are questionable or of only minor value, they should be moved to a working DD with an appropriate" >> "$nwserver"_sw_dvs_not_on_web.txt
print "DD description so that data can be better identified as questionable or marginally useful data." >> "$nwserver"_sw_dvs_not_on_web.txt
print "" >> "$nwserver"_sw_dvs_not_on_web.txt
print "" >> "$nwserver"_sw_dvs_not_on_web.txt
/usr/local/bin/headchg -del < mdvsc | wc -l | read numc
print "THE FOLLOWING DAILY SEDIMENT CONCENTRATION DATA DO NOT APPEAR TO BE ON NWISWEB:" >> "$nwserver"_sw_dvs_not_on_web.txt
print "$numc stations/DDs/stats detected" >> "$nwserver"_sw_dvs_not_on_web.txt
/usr/local/bin/ptbl -p0 -l132 < mdvsc  >> "$nwserver"_sw_dvs_not_on_web.txt
print "" >> "$nwserver"_sw_dvs_not_on_web.txt
print "" >> "$nwserver"_sw_dvs_not_on_web.txt
print "" >> "$nwserver"_sw_dvs_not_on_web.txt
/usr/local/bin/headchg -del < mdvsl | wc -l | read numl
print "THE FOLLOWING DAILY SEDIMENT LOAD DATA DO NOT APPEAR TO BE ON NWISWEB:" >> "$nwserver"_sw_dvs_not_on_web.txt
print "$numl stations/DDs/stats detected" >> "$nwserver"_sw_dvs_not_on_web.txt
/usr/local/bin/ptbl -p0 -l132 < mdvsl  >> "$nwserver"_sw_dvs_not_on_web.txt
print "" >> "$nwserver"_sw_dvs_not_on_web.txt
print "" >> "$nwserver"_sw_dvs_not_on_web.txt
print "" >> "$nwserver"_sw_dvs_not_on_web.txt
/usr/local/bin/headchg -del < mdvsq | wc -l | read numq
print "THE FOLLOWING DAILY WATER DISCHARGE DATA DO NOT APPEAR TO BE ON NWISWEB:" >> "$nwserver"_sw_dvs_not_on_web.txt
print "$numq stations/DDs/stats detected" >> "$nwserver"_sw_dvs_not_on_web.txt
/usr/local/bin/ptbl -p0 -l132 < mdvsq  >> "$nwserver"_sw_dvs_not_on_web.txt
print "" >> "$nwserver"_sw_dvs_not_on_web.txt
print "" >> "$nwserver"_sw_dvs_not_on_web.txt
print "" >> "$nwserver"_sw_dvs_not_on_web.txt
/usr/local/bin/headchg -del < mdvsp | wc -l | read nump
print "THE FOLLOWING DAILY PRECIPITATION DATA DO NOT APPEAR TO BE ON NWISWEB:" >> "$nwserver"_sw_dvs_not_on_web.txt
print "$nump stations/DDs/stats detected" >> "$nwserver"_sw_dvs_not_on_web.txt
/usr/local/bin/ptbl -p0 -l132 < mdvsp  >> "$nwserver"_sw_dvs_not_on_web.txt
print "" >> "$nwserver"_sw_dvs_not_on_web.txt
print "" >> "$nwserver"_sw_dvs_not_on_web.txt
print "" >> "$nwserver"_sw_dvs_not_on_web.txt
print "The following sections are for reference and are what the above output was based on:" >> "$nwserver"_sw_dvs_not_on_web.txt
print "" >> "$nwserver"_sw_dvs_not_on_web.txt
print "" >> "$nwserver"_sw_dvs_not_on_web.txt
cat tmp.seddvs.out >>  "$nwserver"_sw_dvs_not_on_web.txt
print ""
print "Output is in file "$nwserver"_sw_dvs_not_on_web.txt"
print ""
print "Script done, exiting..."
exit
