#!/opt/conda/envs/ciao-4.17.0/bin/python3
# coding: utf-8
# 
#  Copyright (C) 2012-2014,2017,2019,2022  Smithsonian Astrophysical Observatory
#
#
#  This program is free software; you can redistribute it and/or modify
#  it under the terms of the GNU General Public License as published by
#  the Free Software Foundation; either version 3 of the License, or
#  (at your option) any later version.
#
#  This program is distributed in the hope that it will be useful,
#  but WITHOUT ANY WARRANTY; without even the implied warranty of
#  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
#  GNU General Public License for more details.
#
#  You should have received a copy of the GNU General Public License along
#  with this program; if not, write to the Free Software Foundation, Inc.,
#  51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
#


# 11/2012 - initial python version 
# 3/2013-if missing READMODE, no need to error out for HRC.
#       -for pipeline, add 2 pars: temproot & keepexit.
#        If keepexit=yes, create a file temproot_tgdetect2.exit 
#        to indicate whether src found (0=found;  2=not_found).
#       -add a new key TGMETHOD w/ values {TGDETECT FINDZO  NONE}
# 10/1/2013 - bug 13729_I: if no evt data, run tgdetect.
# 10/2/2013 - bug 13729_II: if tg_findzo failed, run tgdetect.
# 2/4/2014-bug 13774: failed if method is tgdet_1 and infile
#          has dmfilter (REF: tgdet1-bug-5786). We'll have a 
#          workaround in tgdet-2 for now.
# 6/27/2014-bug 13774 again: failed if infile has dmfilter.
# 3/1/2017 -bug JIRA CIAO-14 :  problem with python 3.5.1 ( byte .vs. string ; etc.. ) 

import os
import sys
import paramio as pio
from math import sqrt
from cxcdm import *       
import subprocess as subp
import glob

'''Use the history module to add HISTORY records to crate'''
from pycrates import read_file 
from pycrates import add_history, CrateKey
from history import HistoryRecord

class _TDET2() :

   # 6/2014 ( bug 13774 )
   # if fName (infile) has dmfilters, use dmcopy to create an intermediate file.
   #
   def remove_infile_filters( s, fName ) :
      outdir =  os.path.dirname( s.outfile )
      tmpfile = os.path.basename( fName ) # remove path but keep filters if any. 
      tmpWd =  tmpfile.split("[")

      s.filterFlag = 0           # no dmfilter 
      new_infile = fName         # original input file name
      if len(tmpWd) > 1 :
         # dmfilter is found
         s.filterFlag = 1
         nono = "tmpzz_"+tmpWd[0]              # infile w/ no path, no filter
         if len(outdir) == 0 :
            outdir =  "."     # outfile has no path specified;
         new_infile = "/".join((outdir,nono))  # intermed. infile

         # infile 'fName' has path and filter;
         cmd = "infile='{0}' outfile={1} clob=yes".format( fName, new_infile)
         syserr,subpOut=s.exe_task_output( "dmcopy", cmd)
         if syserr != 0 :                      # dmcopy failed
            s.display_msg("ERROR: dmcopy cmd failed : \n  dmcopy {0}\n   stop ! \n".format(cmd))
            os._exit(1)     # error_out
      return new_infile 



   def get_params( s, tool, mode="rw", args=None ):
       if None != args:
          try:
             pf = pio.paramopen( tool, mode, args )
          except:
             s.error_out("ERROR: problem with paramopen. \n" )
       else:
          try:
             pf = pio.paramopen( tool, mode )
          except:
             s.error_out("ERROR: problem with paramopen. \n" )

       # Get parameter name/values in order stored in parameter file
       s.pnames= pio.plist( pf )
       s.pvals = [ pio.pget(pf, pp) for pp in s.pnames]

       s.pars = {}
       s.pars = dict(list(zip(s.pnames,s.pvals)))
       pio.paramclose( pf )

       s.infile=s.pars["infile"]
       s.outfile=s.pars["outfile"]
       s.zo_pos_x=s.pars["zo_pos_x"]
       s.zo_pos_y=s.pars["zo_pos_y"]
       s.unlearn_tg_findzo=s.pars["unlearn_tg_findzo"].upper()    # YES/NO
       s.unlearn_tgdetect=s.pars["unlearn_tgdetect"].upper()
       s.verbose=int(s.pars["verbose"])
       s.keepexit=s.pars["keepexit"]
       s.temproot=s.pars["temproot"]
       s.clobber=s.pars["clobber"].lower () == 'yes'

   ## execute task (cmd); if there's a problem, error_out w/ message
   #
   def exe_task (s, tool, cmd, clean="YES" ) :

      if clean == "YES" :
         try:
            pio.punlearn( tool )
         except:
            s.error_out("ERROR: problem with 'punlearn "+tool+"'.\n" )

      status = os.system ( tool+" "+cmd )
      if status != 0  :
         s.error_out( "problem with running "+tool+".\n")


   # execute task (cmd); 
   #  -If cmd failed to run :  don't error_out
   #  -If no errors and no warnings:  expect syserr=0, len(subpOut)=0
   #  -If any warning or any error, display them !
   #
   def exe_task_output (s, tool, cmd, clean="YES" ) :
      if clean == "YES" :
         try:
            pio.punlearn( tool )
         except:
            s.error_out("ERROR: problem with 'punlearn "+tool+"'.\n" )

      cmd = tool+" "+cmd+" 2>&1"  # capture err msg in CalledProcessError.output

      syserr = 0 

      try:
         subpOut = subp.check_output( cmd, shell=True, stderr=None )
      except subp.CalledProcessError as e:
         syserr = e.returncode
         subpOut = e.output
         pass

      if len(subpOut) > 0 :
         TOOL = tool.upper()
         if syserr == 0 :     # just warnings
            s.display_msg ( "\n"+TOOL+"  warnings :" )
         else :
            s.display_msg ( "\n"+TOOL+"  failed with errors :" )

         # display msgs 
         tmpSubp = subpOut.decode(encoding='UTF-8')   # byte to str ; 3/1/2017
         subpOut = tmpSubp
         for msg in subpOut.split("\n") :
            s.display_msg ( "    "+msg )

      return syserr,subpOut

   # end: exe_task_output



   ## display message
   def display_msg( s, msg ) :
      sys.stderr.write( msg +"\n" )
      sys.stderr.flush()      # 5/2019: add; PTPL-87;

   ## error out w/ message
   def error_out( s, msg ) :
      s.display_msg( msg )
      os._exit(1)

   #
   ## read key values from the input evt1 file
   #
   def evtKeys( s ) :
     try :
        s.inBlk = dmTableOpen( s.infile)
     except :
        s.error_out("ERROR: can't open the table file "+s.infile+".\n" )

     s.k_v = {}
     list_s = ["GRATING","DETNAM","READMODE"]
     for kp in list_s : 
       try :
          _dd, _vv  = dmKeyRead( s.inBlk, kp )   # '_vv' is in string
          s.k_v[ kp ] = _vv.upper().decode(encoding='UTF-8')   # byte -> str ; 3/1/2017
       except :
          if kp != "READMODE" :                  
             s.error_out("ERROR: "+kp+" is missing in '" + s.infile + "'\n") ; 3/1/2017
          else :
             s.k_v[ kp ] = "MISSING"

     ## 3.3.2,  zo_pos_x,y :  both must be Default or both must be a real number
     s.verify_keys()

     dmDatasetClose( dmBlockGetDataset( s.inBlk ))

   # end: evtKeys

   #
   ## 3.3.3-5 verify keys in evt1 file 
   #
   def verify_keys ( s ) :
      #  kmethod can only be set to tg_findzo or tgdetect
      s.kmethod = "tg_findzo"      # initial
       
      evtN=int(subp.Popen(['dmlist {0} counts'.format(s.infile)], stdout=subp.PIPE,shell=True).communicate()[0])

      if evtN <= 0 :
         s.kmethod = "tgdetect"    # bug 13729_I
      else :
         ## 3.3.5 :   method=tgdetect if GRATING is NONE
         uv = s.k_v[ "GRATING" ]
         if  uv=="NONE" :
            s.kmethod = "tgdetect"

         ## 3.3.4:   method=tgdetect if DETNAM/INSTRUMENT is HRC or ACIS-I
         uv = s.k_v[ "DETNAM" ]
         if (uv.find("ACIS") != -1 ) :           # acis
            if s.k_v["READMODE"] == "MISSING" :
               s.error_out("ERROR: READMODE is missing in '"+ s.infile +"'\n")   # 3/2013 + 3/1/2017
            if uv.find("0")>-1 or uv.find("1")>-1 or uv.find("2")>-1 or uv.find("3")>-1:
               s.kmethod = "tgdetect"            #acis-I
         else :
            s.kmethod = "tgdetect"               # hrc

         ## 3.3.3 :   method=tgdetect if READMODE is not TIMED
         uv = s.k_v[ "READMODE" ]
         if uv != "TIMED" :
            s.kmethod = "tgdetect"

      # end: if evtN checking


      ## 3.3.2 : 
      s.check_zo_pos_xy()

   # end: verify_keys()


   ## 3.3.2 :  parameters "zo_pos_x,y" MUST both set to a real number 
   ##          or both set to 'default'
   def check_zo_pos_xy (s) :
      # -- check param("zo_pos_x/_y")
      def zo_xy_par ( parStr ) :
         flag = 0 
         # -- set flag to 1 if parStr is a real number
         try:
            aa = float( parStr )   # numerical
            flag = 1               # parStr is a real number 
         except :                  # parStr is not numerical
            aa = 0
            if ( parStr.upper()=="DEFAULT" ):
               flag = 2               # parStr is 'default'
               
         return aa, flag 

      # check the params "zo_pos_x & _y"
      ax, flag1 = zo_xy_par( s.zo_pos_x )
      ay, flag2 = zo_xy_par( s.zo_pos_y )
      if not ( ((flag1==1) and (flag2==1)) or ((flag1==2) and (flag2==2)) ) :
         s.error_out( "ERROR: parameters (zo_pos_x,y) must be either both set to a real number, or both set to default.\n")
       
      if ((flag1==2) and (flag2==2)) :
         ## 'tgdetect' expects lower case  
         s.zo_pos_x  = s.zo_pos_x.lower()
         s.zo_pos_y  = s.zo_pos_y.lower()
   # end: check_zo_pos_xy

   ## if clobber is yes and the outfile exists, error out.
   #
   def check_clob( s, outfile ) :
      if os.path.exists (outfile):
         if s.clobber:
            s.remove_files( outfile )
         else:
            s.error_out ('ERROR: {file} exists and clobber is set to no.\n'.format (file=outfile))

   ## is it a blank string ?
   def isBlank( s, ss ) :
      if len(ss.strip(" ")) == 0 :
         return True
      else :
         return False
   ## remove a list of files
   def remove_files( s, List ) :
      for FN in List.split() :
         for fn in glob.glob( FN ):
            if s.isBlank(fn) == False :
               try:
                  os.unlink( fn )
               except :
                  pass
#end: _TDET2


class TDET2(_TDET2) :
   def __init__(s, tname ) :
      s.get_params( tname, "rw", sys.argv )       

      s.infile = s.remove_infile_filters( s.infile )   # 6/2014

      s.tname = tname

      s.check_clob (s.outfile)

      ## 2.3.3. Read the header and data of the tg findzo src1a file
      #   ( including TIMEDEL, EXPSOURE, NET_COUNTS, COUNT_TG )
      #   and set exposure, counts, and rate values:
      s.evtKeys()

   def add_history_to_file(s, file_name, tool_name, param_names, param_values):
      if os.path.exists( file_name) == False : 
         return 
      crate = read_file(file_name, mode="rw")
    
      histnum = crate.get_key("HISTNUM")
      if histnum is None:
         histnum = CrateKey()
         histnum.name = "HISTNUM"
         histnum.value = 1
         crate.add_key(histnum)
        
      startat = histnum.value
      startat = 1 if startat is None else startat

      hh = HistoryRecord(tool=tool_name,param_names=param_names,
                         param_values=param_values, asc_start=startat)    
      add_history(crate, hh) 
    
      # Increment HISTNUM for new lines
      hlen = hh.as_ASC_FITS().split("\n")
      histnum.value = startat+len(hlen)

      crate.write() 
   # end: add_history_to_file  

#end: TDET2

# main code 
def main() :
   p = TDET2("tgdetect2")
      
   choice="unknown"     # don't initialize it to tgdetect

   if ( p.kmethod=="tg_findzo" ) :

      ## 3.3.6
      cmd="infile={0} outfile={1} zo_pos_x={2} zo_pos_y={3} clob=yes".format(p.infile,p.outfile,p.zo_pos_x,p.zo_pos_y)

      # 10/2/2013
      syserr,subpOut=p.exe_task_output( "tg_findzo", cmd, p.unlearn_tg_findzo )
      if syserr != 0 :     # tg_findzo failed 
         p.display_msg ( "WARNING: tg_findzo failed, continuing with TGDETECT")
         choice = "tgdetect"      # bug 13729_II



      if choice == "unknown" :    # 10/2/2013

         ## 3.3.7: run tg_choose_method with src1a file.
         cmd="infile={0}".format( p.outfile )
         p.exe_task( "tg_choose_method", cmd )       # punlearn

         ## 3.3.8: if  choice='tgdetect', then run tgdetect 
         choice=pio.pget( "tg_choose_method", "method" )

   # end:  kmethod="tg_findzo"


   ## run tgdetect : 
   if (( p.kmethod=="tgdetect" ) or ( choice=="tgdetect")) :

      #  6/2014 - moved the previous bugfix here to remove_infile_filters()

      ## 3.3.3  3.3.4  3.3.5  3.3.8
      #
      # JCC: If we don't want to punlearn tgdetect, then get 
      #      'OBI_srclist_file' from 'tgdetect.par'. This is
      #      'tgdetect' input list file 'src1a.lis' and it is ok
      #      if the file is empty or the file is not found.
      #
      osf="NONE"         # initialize to the default value
      if ( p.unlearn_tgdetect == "NO" ) :
         osf=pio.pget( "tgdetect", "OBI_srclist_file" )

      ## 3/2013-make sure the file tgdetect.exit NOT created (ie. keepexit=no)
      ##        and don't reset tgdetect's temproot here. 
      cmd="infile={0} outfile={1} zo_pos_x={2} zo_pos_y={3} OBI_srclist_file={4} keepexit=no clob=yes".format(p.infile,p.outfile,p.zo_pos_x,p.zo_pos_y,osf)


      ## 10/2/2013
      syserr,subpOut = p.exe_task_output( "tgdetect", cmd, p.unlearn_tgdetect ) 
      if syserr != 0 :   # tgdetect failed 
         os._exit(1)     # error_out 

   # end:  kmethod="tgdetect"  or choice=tgdetect



   # 3/2013 - add keyword  TGMETHOD = {TGDETECT  FINDZO  NONE}
   if ((p.kmethod=="tgdetect") or (choice=="tgdetect")) :
      vvv="TGDETECT"
   else :
      vvv="FINDZO"



   # 3/2013-if keepexit=yes, create tgdetect2.exit w/ 0=src-found; 2=src-not-found;
   srcN=int(subp.Popen(['dmlist {0}"[NET_COUNTS>0]" counts'.format(p.outfile)], stdout=subp.PIPE,shell=True).communicate()[0])



   # 4/8/2013 - if src1a is from FINDZO, continue checking on COUNT_ST & COUNT_TG 
   if (( srcN < 1 ) and ( vvv=="FINDZO") ) :
      COUNT_ST=int(subp.Popen(['dmkeypar {0} COUNT_ST echo+'.format(p.outfile)], stdout=subp.PIPE,shell=True).communicate()[0])
      COUNT_TG=int(subp.Popen(['dmkeypar {0} COUNT_TG echo+'.format(p.outfile)], stdout=subp.PIPE,shell=True).communicate()[0])
      sumCnt=COUNT_ST+COUNT_TG  #findzo: sumCnt=NET_COUNTS+COUNT_ST+COUNT_TG; NET_COUNTS=0 here.
      if sumCnt > 0  :
         srcN = 1      #   reset srcN to a positive number


         
   if (p.keepexit.lower() != "no") :
      exitF=p.temproot+"_tgdetect2.exit"
      p.remove_files( exitF ) 
      if ( srcN < 1 ) :
         subp.Popen(['echo 2 > {0}'.format(exitF)],stdout=subp.PIPE,shell=True).communicate()[0]
      else :
         subp.Popen(['echo 0 > {0}'.format(exitF)],stdout=subp.PIPE,shell=True).communicate()[0]

   # 6/2014
   if p.filterFlag == 1 :     # an intermediate infile created due to dmfilters
      p.remove_files( p.infile) 

   # 3/2013 - add keyword  TGMETHOD = {TGDETECT  FINDZO  NONE}
   if ( srcN < 1 ) :
      vvv="NONE"
   cmd="infile={0} filelist= oper=add key=TGMETHOD value={1} comment='Method used to create src1a file'".format(p.outfile, vvv)
   p.exe_task("dmhedit", cmd)

   # add history  
   p.add_history_to_file(p.outfile, "tgdetect2", p.pnames, p.pvals)

   if ( p.verbose > 0 ): 
      if (( p.kmethod=="tgdetect" ) or ( choice=="tgdetect")) :
         p.display_msg("\n The output src1a file is produced by tgdetect.\n")
      else :
         p.display_msg("\n The output src1a file is produced by tg_findzo.\n")

   os._exit(0)

if __name__=="__main__":
    main()
