#!/usr/bin/env python

#
# Copyright (C) 2018-2024
# Smithsonian Astrophysical Observatory
#
#
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 2 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
# 02110-1301 USA.
#

__toolname__ = "blanksky_sample"
__revision__  = "18 March 2024"

import os
import sys
import tempfile

import paramio
import stk
import pycrates as pcr

from ciao_contrib._tools import fileio, utils

from ciao_contrib.param_wrapper import open_param_file
from ciao_contrib.logger_wrapper import initialize_logger, make_verbose_level, set_verbosity, handle_ciao_errors

from ciao_contrib.runtool import make_tool, new_pfiles_environment, add_tool_history

from ciao_contrib.parallel_wrapper import parallel_pool_futures
from sherpa.utils import parallel_map
from multiprocessing import cpu_count

#############################################################################
#############################################################################

# Set up the logging/verbose code
initialize_logger(__toolname__)

# Use v<n> to display messages at the given verbose level.
v0 = make_verbose_level(__toolname__, 0)
v1 = make_verbose_level(__toolname__, 1)
v2 = make_verbose_level(__toolname__, 2)
v3 = make_verbose_level(__toolname__, 3)
v4 = make_verbose_level(__toolname__, 4)
v5 = make_verbose_level(__toolname__, 5)


class ScriptError(RuntimeError):
    """Error found during running the script. This class is introduced
    in case there is a need to catch such an error and deal with it
    appropriately (e.g. recognize it as distinct from an error raised
    by the code).
    """
    pass


###############################################


def get_par(argv):
    """ 
    Get data_products parameters from parameter file
    """

    pfile = open_param_file(argv,toolname=__toolname__)["fp"]

    # Common parameters:
    params: dict = {}
    pars: dict = {}

    # load all parameters to dictionary
    pars["infile"] = params["infile"] = paramio.pgetstr(pfile,"infile")
    pars["bkgfile"] = params["bkgfile"] = paramio.pgetstr(pfile,"bkgfile")
    pars["bkgout"] = params["bkgout"] = paramio.pgetstr(pfile,"bkgout")
    pars["psf_bkg_out"] = params["psf_bkg_out"] = paramio.pgetstr(pfile,"psf_bkg_out")
    #params["rate"] = paramio.pgetd(pfile,"rate")
    pars["regionfile"] = params["regionfile"] = paramio.pgetstr(pfile,"regionfile")
    pars["fill_out"] = params["fill_out"] = paramio.pgetstr(pfile,"fill_out")
    pars["reproject"] = params["reproject"] = paramio.pgetstr(pfile,"reproject")
    pars["asolfile"] = params["asol"] = paramio.pgetstr(pfile,"asolfile")
    pars["tmpdir"] = params["tmpdir"] = paramio.pgetstr(pfile,"tmpdir")
    pars["random"] = params["randomseed"] = paramio.pgeti(pfile,"random")
    pars["verbose"] = params["verbose"] = paramio.pgeti(pfile,"verbose")
    pars["clobber"] = params["clobber"] = paramio.pgetstr(pfile, "clobber")
    pars["mode"] = params["mode"] = paramio.pgetstr(pfile, "mode")

    ## check and modify parameters
    ################################

    ## error out if there are spaces in absolute paths of various parameters
    checklist_spaces = [(params["infile"], "source events"),
                        (params["bkgfile"], "blanksky background"),
                        (params["bkgout"], "sampled background output"),
                        (params["psf_bkg_out"], "PSF+background output"),
                        (params["regionfile"], "region"),
                        (params["fill_out"], "events substituted output")]
    checklist_spaces.extend([(asol, "aspect solution") for asol in params["asol"]])

    for fn,filetype in checklist_spaces:
        _check_no_spaces(fn,filetype)

    # check if produced "dmfilth"ed events file will be made
    if params["regionfile"] != "" and params["fill_out"] == "":
        raise ScriptError("input 'regionfile' specified, an output 'fill_out' \
file must be specified")

    if params["regionfile"] == "" and params["fill_out"] != "":
        raise ScriptError("'fill_out' output file specified, an input 'regionfile' \
must also be specified")

    # input PSF file
    if params["infile"] == "":
        raise ScriptError("Input events file must be specified.")
    if params["infile"].startswith("@"):
        raise ScriptError("'infile' does not support stacks.")

    params["infile_filter"] = fileio.get_filter(params["infile"])
    params["infile"] = fileio.get_file(params["infile"])

    # input blanksky background file
    if params["bkgfile"] == "":
        raise ScriptError("Input blanksky background file must be specified.")
    if params["bkgfile"].startswith("@"):
        raise ScriptError("Input blanksky background stacks not supported.")

    params["bkgfile_filter"] = fileio.get_filter(params["bkgfile"])
    params["bkgfile"] = fileio.get_file(params["bkgfile"])

    # output blanksky sampled file name
    if params["bkgout"] == "":
        raise ScriptError("Please specify an output background file name.")

    params["bkgoutdir"],bkgoutfile = utils.split_outroot(params["bkgout"])

    if params["bkgout"].endswith("_"):
        params["bkgout"] = bkgoutfile
    else:
        params["bkgout"] = bkgoutfile.rstrip("_")

    # check if output directory is writable
    fileio.validate_outdir(params["bkgoutdir"])

    # output PSF+bkg file name
    if params["psf_bkg_out"] != "":
        psf_bkg_outdir,psf_bkg_outfile = utils.split_outroot(params["psf_bkg_out"])

        if params["psf_bkg_out"].endswith("_"):
            params["psf_bkg_out"] = psf_bkg_outfile
        else:
            params["psf_bkg_out"] = psf_bkg_outfile.rstrip("_")

        # check if output directory is writable
        params["psf_bkg_outdir"] = psf_bkg_outdir
        fileio.validate_outdir(params["psf_bkg_outdir"])

    # input region file
    if params["regionfile"] != "":
        params["regionfile"] = fileio.get_file(params["regionfile"])

    # output fill_out file name
    if params["fill_out"] != "":
        params["fill_outdir"],fill_outfile = utils.split_outroot(params["fill_out"])

        if params["fill_out"].endswith("_"):
            params["fill_out"] = fill_outfile
        else:
            params["fill_out"] = fill_outfile.rstrip("_")

        # check if output directory is writable
        fileio.validate_outdir(params["fill_outdir"])

    # aspect solution files, listed in quotes.
    if params["asol"] != "":
        params["asol"] = ",".join(stk.build(params["asol"]))


    # close parameters block after validation
    paramio.paramclose(pfile)

    v3(f"  Parameters: {params}")

    return params,pars



def _check_no_spaces(fn,filetype):
    abspath = os.path.abspath(fn)

    if " " in abspath:
        raise IOError(f"The absolute path for the {filetype} file, '{abspath}', cannot contain any spaces")



def _get_nevt(fn):
    """get number of events"""

    cr = pcr.read_file(fn)

    nevt = cr.get_nrows()

    del cr

    return nevt



def _cols2int4(fn,tmpdir):
    """Convert the PI and PHA columns in the blanksky file from Int2 to Int4 data types"""

    dmcopy = make_tool("dmcopy")
    dmtcalc = make_tool("dmtcalc")

    # check that column datatype
    cr = pcr.read_file(f"{fn}[cols pi,pha][#row=0]")

    pi_type = cr.get_column("pi").values.dtype.name
    pha_type = cr.get_column("pha").values.dtype.name

    del cr

    if False in [pi_type=="int64",pha_type=="int64"]:
        v2("Converting PI and PHA column integer type for compatibility...")

        if [True,True] == [pi_type!="int64",pha_type!="int64"]:
            convert_col = "tmppha=(long)pha,tmppi=(long)pi"
            tmpcol = "pha=tmppha,pi=tmppi"
            col_replace = "[cols -pha,-pi]"
            tmpcol_del = "[cols -tmppha,-tmppi]"
        else:
            if pi_type != "int64":
                convert_col = "tmppi=(long)pi"
                tmpcol = "pi=tmppi"
                col_replace = "[cols -pi]"
                tmpcol_del = "[cols -tmppi]"
            elif pha_type != "int64":
                convert_col = "tmppha=(long)pha"
                tmpcol = "pha=tmppha"
                col_replace = "[cols -pha]"
                tmpcol_del = "[cols -tmppha]"

        with tempfile.NamedTemporaryFile(dir=tmpdir) as tmp1, tempfile.NamedTemporaryFile(dir=tmpdir) as tmp2:
            dmtcalc.punlearn()
            dmtcalc.infile = fn
            dmtcalc.outfile = tmp1.name
            dmtcalc.expression = convert_col
            dmtcalc.verbose = 0
            dmtcalc.clobber= True
            dmtcalc()

            dmtcalc.punlearn()
            dmtcalc.infile = f"{tmp1.name}{col_replace}"
            dmtcalc.outfile = tmp2.name
            dmtcalc.expression = tmpcol
            dmtcalc.verbose = 0
            dmtcalc.clobber= True
            dmtcalc()

            dmcopy.punlearn()
            dmcopy.infile = f"{tmp2.name}{tmpcol_del}"
            dmcopy.outfile = fn
            dmcopy.verbose = 0
            dmcopy.clobber = True
            dmcopy()



def _convert_ppr_cols(infile,outfile,instrument,tmpdir):
    dmcopy = make_tool("dmcopy")
    dmtcalc = make_tool("dmtcalc")
    dmhedit = make_tool("dmhedit")

    with tempfile.NamedTemporaryFile(dir=tmpdir) as tmp1, \
         tempfile.NamedTemporaryFile(dir=tmpdir) as tmp2:

        # copy the DETPOS column to a new DET column, for merging purposes
        dmtcalc.punlearn()
        dmtcalc.infile = infile
        dmtcalc.outfile = tmp1.name
        dmtcalc.expression = "tmpx=(float)x,tmpy=(float)y,tmpdetx=(float)detx,tmpdety=(float)dety,tmpchipx=(short)chipx,tmpchipy=(short)chipy"
        dmtcalc.verbose = 0
        dmtcalc.clobber= True
        dmtcalc()

        dmtcalc.punlearn()
        dmtcalc.infile = f"{tmp1.name}[cols -sky,-chip,-detpos]"
        dmtcalc.outfile = tmp2.name
        dmtcalc.expression = "x=tmpx,y=tmpy,detx=tmpdetx,dety=tmpdety,chipx=tmpchipx,chipy=tmpchipy"
        dmtcalc.verbose = 0
        dmtcalc.clobber= True
        dmtcalc()

        dmhedit.punlearn()
        dmhedit.infile = tmp2.name
        dmhedit.operation = "add"
        dmhedit.key = "MTYPE1"
        dmhedit.value = "chip"
        dmhedit.verbose = 0
        dmhedit()

        dmhedit.punlearn()
        dmhedit.infile = tmp2.name
        dmhedit.operation = "add"
        dmhedit.key = "MFORM1"
        dmhedit.value = "chipx,chipy"
        dmhedit.verbose = 0
        dmhedit()

        dmhedit.punlearn()
        dmhedit.infile = tmp2.name
        dmhedit.operation = "add"
        dmhedit.key = "MTYPE2"
        dmhedit.value = "det"
        dmhedit.verbose = 0
        dmhedit()

        dmhedit.punlearn()
        dmhedit.infile = tmp2.name
        dmhedit.operation = "add"
        dmhedit.key = "MFORM2"
        dmhedit.value = "detx,dety"
        dmhedit.verbose = 0
        dmhedit()

        dmhedit.punlearn()
        dmhedit.infile = tmp2.name
        dmhedit.operation = "add"
        dmhedit.key = "MTYPE3"
        dmhedit.value = "sky"
        dmhedit.verbose = 0
        dmhedit()

        dmhedit.punlearn()
        dmhedit.infile = tmp2.name
        dmhedit.operation = "add"
        dmhedit.key = "MFORM3"
        dmhedit.value = "x,y"
        dmhedit.verbose = 0
        dmhedit()

        for key in ["X","Y","DETX","DETY"]:
            keynum = _structural_kw(tmp2.name,key)

            if keynum is not None:
                dmhedit.punlearn()
                dmhedit.infile = tmp2.name
                dmhedit.operation = "add"
                dmhedit.key = f"TLMIN{keynum}"
                dmhedit.value = "0.5"
                dmhedit.verbose = 0
                dmhedit()

                dmhedit.punlearn()
                dmhedit.infile = tmp2.name
                dmhedit.operation = "add"
                dmhedit.key = f"TLMAX{keynum}"
                dmhedit.value = "8192.5"
                dmhedit.verbose = 0
                dmhedit()

        if instrument == "HRC":
            # ppr output columns is agnostic on instrument and just uses CCD_ID, which
            # isn't appropriate for HRC data, which uses CHIP_ID

            dmtcalc.punlearn()
            dmtcalc.infile = tmp2.name
            dmtcalc.outfile = tmp1.name
            dmtcalc.expression = "chip_id=ccd_id"
            dmtcalc.verbose = 0
            dmtcalc.clobber= True
            dmtcalc()

            dmcopy.punlearn()
            dmcopy.infile = f"{tmp1.name}[cols -ccd_id]"
            dmcopy.outfile = outfile
            dmcopy.verbose = 0
            dmcopy.clobber = True
            dmcopy()

        else:
            dmcopy.punlearn()
            dmcopy.infile = f"{tmp2.name}"
            dmcopy.outfile = outfile
            dmcopy.verbose = 0
            dmcopy.clobber = True
            dmcopy()



def _structural_kw(fn,keyval):

    dmlist = make_tool("dmlist")

    dmlist.punlearn()
    dmlist.infile = fn
    dmlist.opt = "keys,raw"
    dmlist.verbose = 0

    col = dmlist()
    col = [ln for ln in col.split("\n") if "TTYPE" in ln]
    col = [i.split("=") for i in col]

    col_dict = { d[0].replace(" ","").split("*")[-1] : d[1].replace(" ","").split("/")[0] for d in col }

    for k,v in col_dict.items():
        if v.lower() == keyval.lower():
            return  int(k.replace("TTYPE",""))

    return None



def _evt_type(fn):
    """
    ID infile input type.
    # MARX: HDUNAME=EVENTS, DATACLAS=SIMULATED
    # PPR: HDUNAME=RAYEVENTS, DATACLAS=OBSERVED
    # Obs: HDUNAME=EVENTS, DATACLAS=OBSERVED
    """

    kw = fileio.get_keys_from_file(fn)

    hduclass = kw["HDUNAME"]
    dataclass = kw["DATACLAS"]

    status = (hduclass == "EVENTS", dataclass == "OBSERVED")

    if status == (True,True):
        evt_type = "obs"
    elif status == (True,False):
        evt_type = "marx"
    elif status == (False,True):
        evt_type = "ppr"
    else:
        evt_type = None

    return evt_type



def _num_pick(args):
    """Pick number of photons to add to the simulation"""

    bkg,outfile,bkgscale,bkgmeth,tobs,tbsky,randomseed,tmpdir,verbose = args

    dmcopy = make_tool("dmcopy")
    dmtcalc = make_tool("dmtcalc")

    with new_pfiles_environment(ardlib=False,copyuser=False), tempfile.NamedTemporaryFile(dir=tmpdir) as def_rand:

        ## get total number of events; the NAXIS2/__NROWS keyword aren't updated if
        ## evt file is filtered
        # n = kw["__NROWS"]
        # n = _get_nevt(bkg)
        # cts = rate*exptime # sample the scaled down background

        # these generate random values between 0 and 1, inclusive
        if randomseed > 0:
            randseed = f"#rand({randomseed})"
        else:
            randseed = "#trand"

        dmtcalc.punlearn()
        dmtcalc.infile = bkg
        dmtcalc.outfile = def_rand.name
        dmtcalc.verbose = verbose
        dmtcalc.clobber = True

        if bkgmeth.lower() == "particle-rate":
            # # get HE filter
            # dmhistory.punlearn()
            # dmhistory.infile = bkg
            # dmhistory.tool = "blanksky"
            # bsky_output = dmhistory().split("\n")[-1].split(" ")[1:]
            #
            # bsky_kw = {}
            # for d in bsky_output:
            #     bsky_kw[d.split("=")[0]] = d.split("=")[-1].replace("\"","")
            #
            # if bsky_kw["bkgparams"] == "default":
            #     efilt = "[energy=9000:12000]"
            # else:
            #     efilt = bsky_kw["bkgparams"]
            #
            # ntot = _get_nevt(bkg)
            # nhe = _get_nevt("{0}{1}".format(bkg,efilt))

            dmtcalc.expression = f"randnum={randseed}*({tbsky}/({tobs}*{bkgscale}))"
        else:
            dmtcalc.expression = f"randnum={randseed}/{bkgscale}"

        dmtcalc()

        dmcopy.punlearn()
        dmcopy.infile = f"{def_rand.name}[randnum=0:1]"
        dmcopy.outfile = outfile # sampled event will have RANDNUM value less than 1
        dmcopy.clobber = True

        dmcopy()



def assign_time(bkg_rand,time_sorted_outfile,kw,randomseed,tmpdir,verbose):
    """Assign some random time during the observation to each photon"""

    t0 = kw["TSTART"]
    t1 = kw["TSTOP"]
    dtcor = kw["DTCOR"]

    t0_corr = t0 + 0.5*(1-dtcor)*(t1-t0)
    t1_corr = t1 - 0.5*(1-dtcor)*(t1-t0)

    dmtcalc = make_tool("dmtcalc")
    dmsort = make_tool("dmsort")
    dmhedit = make_tool("dmhedit")

    with tempfile.NamedTemporaryFile(dir=tmpdir) as time_rand:
        if randomseed > 0:
            randseed = f"#rand({randomseed})"
        else:
            randseed = "#trand"

        dmtcalc.punlearn()

        dmtcalc.infile = bkg_rand
        dmtcalc.outfile = time_rand.name
        #dmtcalc.expression = "time={0}+({1}-{0})*{2}".format(t0,t1,randseed)
        dmtcalc.expression = f"time={t0_corr}+({t1_corr}-{t0_corr})*{randseed}"
        dmtcalc.clobber = True
        dmtcalc.verbose = verbose

        dmtcalc()

        dmsort.punlearn()

        dmsort.infile = time_rand.name
        dmsort.outfile = time_sorted_outfile
        dmsort.keys = "TIME"
        dmsort.clobber = True

        dmsort()

    # update time-related header keywords of the sampled file
    dmhedit.punlearn()
    dmhedit.infile = time_sorted_outfile
    dmhedit.operation = "add"
    dmhedit.verbose = 0

    for tkey in ["ONTIME","LIVETIME","EXPOSURE","TSTART","TSTOP","DTCOR"]:
        dmhedit.key = tkey
        dmhedit.value = kw[tkey]
        dmhedit()

    if kw["INSTRUME"] == "ACIS":
        chips = fileio.get_ccds(bkg_rand)

        for chip in chips:
            dmhedit.key = f"ONTIME{chip}"
            dmhedit.value = kw[f"ONTIME{chip}"]
            dmhedit()

            dmhedit.key = f"LIVTIME{chip}"
            dmhedit.value = kw[f"LIVTIME{chip}"]
            dmhedit()

            dmhedit.key = f"EXPOSUR{chip}"
            dmhedit.value = kw[f"EXPOSUR{chip}"]
            dmhedit()



def sample_chip(bkg,infile,outfile,kw,randomseed,tmpdir,verbose):
    """
    randomly sample the number of events for each chip, proportional to 1/BKGSCALn
    """

    instrument = kw["INSTRUME"]
    bkgmeth = kw["BKGMETH"]

    # determine chips to be used for the image
    if instrument == "HRC":
        bkgscale = kw["BKGSCALE"]

        tobs = fileio.get_keys_from_file(infile)["LIVETIME"]
        tbsky = kw["LIVETIME"]

        _num_pick((bkg,outfile,bkgscale,bkgmeth,tobs,tbsky,randomseed,tmpdir,verbose))

    else:
        chips = fileio.get_ccds(bkg)

        bkg_sample = []
        bkg_tmp = []

        dmmerge = make_tool("dmmerge")

        try:
            for chip in chips:
                bkgtmp = tempfile.NamedTemporaryFile(suffix=f".bkg{chip}",dir=tmpdir)
                bkgscale = kw[f"BKGSCAL{chip}"]
                bkgchipstr = f"{bkg}[ccd_id={chip}]"

                tobs = fileio.get_keys_from_file(infile)[f"LIVTIME{chip}"]
                try:
                    tbsky = kw[f"LIVTIME{chip}"]
                except KeyError:
                    try:
                        tbsky = kw[f"LIVETIM{chip}"]
                    except KeyError:
                        tbsky = kw["LIVETIME"]

                bkg_sample.append((bkgchipstr,bkgtmp.name,bkgscale,bkgmeth,tobs,tbsky,randomseed,tmpdir,verbose))

                bkg_tmp.append(bkgtmp)

            if len(chips) > cpu_count():
                parallel_pool_futures(_num_pick,bkg_sample)
            else:
                parallel_map(_num_pick,bkg_sample)

        finally:
            dmmerge.punlearn()

            dmmerge.infile = [bg.name for bg in bkg_tmp]
            dmmerge.outfile = outfile
            dmmerge.clobber = True
            dmmerge.verbose = verbose

            dmmerge()

            for fn in bkg_tmp:
                fn.close()



@handle_ciao_errors(__toolname__,__revision__)
def doit():
    params,pars = get_par(sys.argv)

    # print script info
    set_verbosity(params["verbose"])
    utils.print_version(__toolname__, __revision__)

    infile = params["infile"]
    bkgfile = params["bkgfile"]
    bkgout = params["bkgout"]
    bkgoutdir = params["bkgoutdir"]
    psf_bkg_out = params["psf_bkg_out"]
    asol = params["asol"]
    reproject = params["reproject"]
    fill_out = params["fill_out"]
    regionfile = params["regionfile"]
    tmpdir = params["tmpdir"]
    seed = params["randomseed"]
    clobber = params["clobber"]
    verbose = params["verbose"]

    dmcopy = make_tool("dmcopy")
    dmmerge = make_tool("dmmerge")
    dmhedit = make_tool("dmhedit")

    ############
    with new_pfiles_environment(ardlib=False), \
         tempfile.NamedTemporaryFile(dir=tmpdir) as tmpin, \
         tempfile.NamedTemporaryFile(dir=tmpdir) as get_rand, \
         tempfile.NamedTemporaryFile(dir=tmpdir) as time_sorted:

        kw_psf = fileio.get_keys_from_file(f"{infile}{params['infile_filter']}")
        kw_bkg = fileio.get_keys_from_file(bkgfile)

        instrument = kw_bkg["INSTRUME"]

        etype = _evt_type(infile)

        # filter CCDs if input is a PSF to what's available in the BKG, since PPR contains
        # all CCD_IDs and MARX is strictly defined between ACIS-I and ACIS-S
        with tempfile.NamedTemporaryFile(dir=tmpdir) as tmpbkg:
            if etype != "obs" and instrument == "ACIS":

                ccd_psf = fileio.get_ccds(infile)
                ccd_bkg = fileio.get_ccds(bkgfile)

                ccd = set(ccd_psf) & set(ccd_bkg)
                ccd = ",".join([str(i) for i in sorted(ccd)])

                dmcopy.punlearn()
                dmcopy.infile = f"{infile}[ccd_id={ccd}]"
                dmcopy.outfile = tmpin.name
                dmcopy.verbose = 0
                dmcopy.clobber = True
                dmcopy()

                dmcopy.punlearn()
                dmcopy.infile = f"{bkgfile}[ccd_id={ccd}]"
                dmcopy.outfile = tmpbkg.name
                dmcopy.verbose = 0
                dmcopy.clobber = True
                dmcopy()

                infile = tmpin.name
                bkgfile = tmpbkg.name

            # check for background CALDB version; any ACIS background before 4.7.5.1
            # will need to convert the PHA and PI datatype which were short integers
            # (Int16/Int2) and match the event files with long integers (Int64/Int4)
            cr_bkg = pcr.read_file(f"{bkgfile}[#row=0]")

            bkg_old = (True in (cr_bkg.pi.values.dtype != "int64",
                                cr_bkg.pha.values.dtype != "int64"))

            del cr_bkg

            # sample background file and assign times to each event
            sample_chip(bkgfile,infile,get_rand.name,kw_bkg,seed,tmpdir,verbose)
            assign_time(get_rand.name,time_sorted.name,kw_psf,seed,tmpdir,verbose)

        if etype != "ppr":
            if instrument == "ACIS":
                if bkg_old:
                    _cols2int4(time_sorted.name,tmpdir) # convert PHA and PI columns to Int4 to match observed evt

        if reproject == "yes":
            reproject_events = make_tool("reproject_events")

            # Reproject the blank sky fields to the same position on the sky as the marx simulation.
            reproject_events.punlearn()

            if instrument == "ACIS":
                reproject_events.infile = f"{time_sorted.name}[cols ccd_id,node_id,chip,det,sky,pha,energy,pi,fltgrade,grade,status,time]"
            else:
                reproject_events.infile = f"{time_sorted.name}[cols chip_id,chip,det,sky,pha,pi,status,time]"

            reproject_events.outfile = f"{bkgoutdir}{bkgout}"
            reproject_events.aspect = asol
            reproject_events.match = infile
            reproject_events.random = seed
            reproject_events.clobber = True
            reproject_events.verbose = verbose

            reproject_events()

        else:
            # if the blank sky file is already reprojected or if the blanksky script was used
            dmcopy.punlearn()

            if instrument == "ACIS":
                dmcopy.infile = f"{time_sorted.name}[cols ccd_id,node_id,chip,det,sky,pha,energy,pi,fltgrade,grade,status,time]"
            else:
                dmcopy.infile = f"{time_sorted.name}[cols chip_id,chip,det,sky,pha,pi,status,time]"

            dmcopy.outfile = bkgoutdir+bkgout
            dmcopy.clobber = clobber
            dmcopy()

        try:
            add_tool_history(bkgoutdir+bkgout,__toolname__,pars)
        except OSError:
            pass

        ## Merge MARX simulation and blank sky fields into a single fits table.
        if psf_bkg_out != "":

            psf_bkg_outdir = params["psf_bkg_outdir"]

            dmmerge.punlearn()

            with tempfile.NamedTemporaryFile(dir=tmpdir) as tmppsf:
                if etype == "ppr":
                    _convert_ppr_cols(infile,tmppsf.name,instrument,tmpdir)
                    dmmerge.infile = f"{tmppsf.name},{bkgoutdir}{bkgout}"

                elif etype in ["obs","marx"]:
                    if (True,True) == (etype=="obs",instrument=="ACIS"):
                        dmmerge.infile = f"{infile},{bkgoutdir}{bkgout}"

                    else:
                        # first need to convert data type for PI and PHA columns
                        dmcopy.punlearn()
                        dmcopy.infile = infile
                        dmcopy.outfile = tmppsf.name
                        dmcopy.verbose = 0
                        dmcopy.clobber = True
                        dmcopy()

                        _cols2int4(tmppsf.name,tmpdir)
                        dmmerge.infile = f"{tmppsf.name},{bkgoutdir}{bkgout}"

                else:
                    v3("'infile' has unrecognized creator")


                if etype == "ppr":
                    if instrument == "ACIS":
                        dmmerge.columnList = "ccd_id,chip,det,sky,energy"
                    else:
                        dmmerge.columnList = "chip_id,chip,det,sky"

                else:
                    if instrument == "ACIS":
                        dmmerge.columnList = "time,ccd_id,node_id,chip,det,sky,pha,energy,pi,fltgrade,grade,status"
                    else:
                        dmmerge.columnList = "time,chip_id,chip,det,sky,pha,status"

                dmmerge.outfile = f"{psf_bkg_outdir}{psf_bkg_out}"
                dmmerge.clobber = clobber
                dmmerge.verbose = verbose

                dmmerge()

                # undo 'merged' keywords in resulting product
                dmhedit.punlearn()
                dmhedit.infile = f"{psf_bkg_outdir}{psf_bkg_out}"
                dmhedit.operation = "add"
                dmhedit.verbose = 0

                for tkey in ["DATACLAS","OBSERVER","TITLE","OBS_ID",
                             "SEQ_NUM","DETNAM","OBJECT","DATAMODE",
                             "RA_TARG","DEC_TARG",
                             "RA_PNT","DEC_PNT","ROLL_PNT",
                             "RA_NOM","DEC_NOM","ROLL_NOM"]:
                    dmhedit.key = tkey
                    dmhedit.value = kw_psf[tkey]
                    dmhedit()

                dmhedit.key = "ASPTYPE"
                dmhedit.op = "delete"
                dmhedit()

            try:
                add_tool_history(f"{psf_bkg_outdir}{psf_bkg_out}", __toolname__, pars)
            except OSError:
                pass

        if fill_out != "":

            fill_outdir = params["fill_outdir"]

            dmmerge.punlearn()

            dmmerge.outfile = fill_outdir+fill_out
            dmmerge.clobber = clobber
            dmmerge.verbose = verbose

            if etype == "ppr":
                with tempfile.NamedTemporaryFile(dir=tmpdir) as tmppsf:
                    if psf_bkg_out == "":
                        _convert_ppr_cols(infile,tmppsf.name,instrument,tmpdir)

                        dmmerge.infile = f"{tmppsf.name}{params['infile_filter']}[exclude sky=region({regionfile})],{bkgoutdir}{bkgout}{params['infile_filter']}[sky=region({regionfile})]"

                    else:
                        dmmerge.infile = f"{tmppsf.name}{params['infile_filter']}[exclude sky=region({regionfile})],{bkgoutdir}{bkgout}{params['infile_filter']}[sky=region({regionfile})]"

                    if instrument == "ACIS":
                        dmmerge.columnList = "ccd_id,chip,det,sky,energy"
                    else:
                        dmmerge.columnList = "chip_id,chip,det,sky"

                    dmmerge()

            else:
                dmmerge.infile = f"{infile}{params['infile_filter']}[exclude sky=region({regionfile})],{bkgoutdir}{bkgout}{params['infile_filter']}[sky=region({regionfile})]"

                if instrument == "ACIS":
                    dmmerge.columnList = "ccd_id,node_id,chip,det,sky,PHA,energy,PI,fltgrade,grade,status,time"
                else:
                    if etype == "obs":
                        dmmerge.columnList = "chip_id,chip,det,sky,pha,pi,status,TIME"
                    elif etype == "marx":
                        dmmerge.columnList = "chip_id,chip,det,sky,pha,status,TIME"
                    else:
                        v3("'infile' has unrecognized creator")

                dmmerge()

            try:
                add_tool_history(f"{fill_outdir}{fill_out}", __toolname__, pars)
            except OSError:
                pass



if __name__ == "__main__":
    doit()
