pro clean_imgdb,sc,tel,d1,d2,removeall=removeall,naxiseq0=naxiseq0,noscript=noscript,noattic=noattic
; $Id: clean_imgdb.pro,v 1.7 2010/06/23 17:25:22 mcnutt Exp $
;
; Project     : STEREO SECCHI
;                   
; Name        : clean_imgdb
;               
; Purpose     : to remove incorrectly named images from database
;               
; Explanation : 
;               
; Use         : IDL> clean_imgdb,'A','HI1','20100510',d2
;    
; Inputs      :   sc = A or B 
;                 tel =HI_1 , HI_2 ,COR2, COR1 or EUVI
;   	     	  d1 = yyyymmdd  first day or full day requested
;    	    	    	    if only d1 is set then the full day is returned
;   	    	  d2 = ending day optional
;               
; Outputs     :   script file for database scc(AorB)cd(machine)yyyymmdd(time).script
;
; Keywords :	removeall : deletes all files from database and write new script for db
;               naxiseq0 NOT USED
;               noscript : do not write db script file only write remove fits file
;               noattic : set if sci attic not available.
;               
; Calls from LASCO : none
;
; Common      : 
;               
; Restrictions: 
;               
; Side effects: 
;               
; Category    : Database 
;               
; Prev. Hist. : None.
;
; Written     : Lynn McNutt, NRL/I2, Aug 2006
;               
; Modified    :
;
; $Log: clean_imgdb.pro,v $
; Revision 1.7  2010/06/23 17:25:22  mcnutt
; added keyword discriptions and change sci attic
;
; Revision 1.6  2009/02/12 15:02:01  mcnutt
; added noattic keyword used when the sci attic files are not available and change order of inputs so d2 is the last one
;
; Revision 1.5  2008/03/07 22:14:43  nathan
; Use find_common.pro and rest_mask.pro to determine missing files rather than
; check each one; use sci (attic) to determine files missing from database rather
; than FITS; updated logfile to give totals per day; TBD is to compare FITS and
; attic files
;
; Revision 1.4  2008/03/05 19:32:01  nathan
; omit realtime fileorig from database validity check; added some logging
;
; Revision 1.3  2007/11/06 16:09:16  mcnutt
; crate yyyymmdd fro day d1 to d2
;
; Revision 1.2  2007/11/06 16:03:44  mcnutt
; adds image to database if missing
;
; Revision 1.1  2007/10/03 20:24:32  mcnutt
; compares fits files to image database and writes scripts to remove exta files from database
;
;
;
common scc_reduce,  lulog, basehdr, nom_hdr, ext_hdr, scch, leapsecs, enums, resetvars, $
    	    	    ipdat, nom_tlr, prevbasename, readfileidline, wavefileidline, $
		    setupfileidline, exposfileidline, maskfileidline, ipfileidline, icerdiv2flag, $
		    rotblidlines,hbtimes,hbtemps,cebtimes,cebtemps, subdir, date_fname_utc, ispb, islz, isrt, aorb, $
		    iserror, ephemfil, att_file
common dbms,ludb, dbfile, delflag, img_seb_hdr, img_seb_hdr_ext, img_cal_bias, $
	    img_cal_led, img_cal_dark, img_hist_cmnt, img_euvi_gt    	
ud1=anytim2utc(d1)
IF n_params() GT 3 THEN ud2=anytim2utc(d2) ELSE ud2=ud1
ud=ud1

IF ud1.mjd GT ud2.mjd THEN incr=-1 ELSE incr=1


FOR mjd=ud1.mjd,ud2.mjd,incr DO BEGIN
;--Test 1 day at a time
ud.mjd=mjd
yyyymmdd=utc2yymmdd(ud,/yyyy)

yyyymmdd=string(yyyymmdd,'(i8.8)')
delfile=getenv('seb_img')+'/db/scc'+strupcase(sc)+'rmfits'+tel+yyyymmdd+'.txt'
outfile=getenv('seb_img')+'/db/scc'+strupcase(sc)+'rmextra'+tel+yyyymmdd+'.script'
logfile=getenv('seb_img')+'/log/scc'+strupcase(sc)+'dbcomp'+tel+yyyymmdd+'.log'
lulog=9
ludb=8

pipeline ='cd'
dbroot=yyyymmdd
get_utc,ustarttime
stt=utc2str(ustarttime,/time_only)
dbroot=yyyymmdd+strmid(stt,0,2)+strmid(stt,3,2)+strmid(stt,6,2)
SPAWN,'hostname',machine,/sh  
 machine = machine(N_ELEMENTS(machine)-1)
src=strmid(machine,0,1)+rstrmid(machine,0,1)
dbfile=concat_dir(getenv('SCC_DB'),'scc'+strupcase(sc)+pipeline+src+dbroot+'sql.script')

case strupcase(tel) of
   'COR1' : detect='2'
   'COR2' : detect='1'
   'EUVI' : detect='3'
   'HI1' : detect='5'
   'HI2' : detect='4'
   'HI_1' : detect='5'
   'HI_2' : detect='4'
endcase
teld=strlowcase(tel)
case strupcase(tel) of
   'HI1' : teld='hi_1'
   'HI2' : teld='hi_2'
   else  : teld=strlowcase(tel)
endcase 

if (strupcase(sc) eq 'A') then $
    	    sccattic=getenv_slash('scia')+strmid(yyyymmdd,2,6)+'/' else $
    	    sccattic=getenv_slash('scib')+strmid(yyyymmdd,2,6)+'/'

;sccattic=getenv('SCC_ATTIC')+'/'+strmid(yyyymmdd,2,6)+'/'

if (strupcase(sc) eq 'A') then aorb='1' else aorb='2'

time1=strmid(yyyymmdd,0,4)+'-'+strmid(yyyymmdd,4,2)+'-'+strmid(yyyymmdd,6,2)+' 00:00:00'
time2=strmid(yyyymmdd,0,4)+'-'+strmid(yyyymmdd,4,2)+'-'+strmid(yyyymmdd,6,2)+' 23:59:59'

;if ~keyword_set(naxiseq0) then nax=' and naxis1 >0' else nax=''
nax=''

;--Get list from database
cmnd='select fileorig,filename,date_obs,date_mod,naxis1 from img_seb_hdr where obsrvtry ='+aorb+' and date_obs between "'+time1+'" and "'+time2+'" and detector ='+detect+' and  downlink > 2 '+nax
print,cmnd
imgdb=SQDB('secchi_flight',cmnd)
;cmnd='select fileorig,date_obs,date_mod from img_seb_hdr_ext where date_obs between "'+time1+'" and "'+time2+'"'
;print,cmnd
;imgdbext=SQDB('secchi_flight',cmnd)
;cmnd='select filename,date_obs,date_mod from img_hist_cmnt where date_obs between "'+time1+'" and "'+time2+'"'
;print,cmnd
;imghist=SQDB('secchi_flight',cmnd)

if(datatype(imgdb) ne 'STR')then begin	;--Make sure database query result is valid

;--Get list of FITS files
ftsfiles0=findfile(getenv('secchi')+'/lz/L0/'+strlowcase(sc)+'/*/'+teld+'/'+yyyymmdd+'/*.fts') 
break_file,ftsfiles0,v1,path,roots,sfxs
ftsfiles=roots+sfxs

if ftsfiles0(0) ne '' then begin
for nfo=0,n_elements(ftsfiles0)-1 do begin
        img=sccreadfits(ftsfiles0[nfo],scch,/nodata)
        if nfo eq 0 then ftsorig=scch.fileorig else ftsorig=[ftsorig,scch.fileorig]
        if nfo eq 0 then ftsdate=scch.date else ftsdate=[ftsdate,scch.date]
endfor


ftsfiles0=ftsfiles0(sort(ftsorig))   
ftsfiles=ftsfiles(sort(ftsorig))
ftsdate=ftsdate(sort(ftsorig))
ftsorig=ftsorig(sort(ftsorig))
uniqcheck=uniq(ftsorig)

tftsfiles=ftsfiles0
if n_Elements(uniqcheck) ne n_Elements(ftsorig) then begin

   openw,rmlun,delfile,/get_lun,width=200
   for nfo=0,n_Elements(uniqcheck)-1 do begin
      zq=where(ftsorig eq ftsorig(uniqcheck(nfo)))
      if n_Elements(zq) gt 1 then begin
        rmf=where(ftsdate(zq) lt max(ftsdate(zq)))
        for np=0,n_Elements(rmf)-1 do begin
	  printf,rmlun,'/bin/rm -f '+tftsfiles(zq(rmf(np)))
          print,'/bin/rm -f '+tftsfiles(zq(rmf(np)))
	  ftsfiles0=ftsfiles0(where(ftsfiles0 ne tftsfiles(zq(rmf(np)))))
          ftsfiles=ftsfiles(where(ftsfiles0 ne tftsfiles(zq(rmf(np)))))
        endfor
      endif
    endfor
    close,rmlun
    free_lun,rmlun
endif
endif


;--Get list of SCI/attic files
td=string((detect+(aorb-1)*5) mod 10,'(i1)')
sci1=file_search(sccattic+'*.??'+td)
sci2=file_search(sccattic+'*.??'+td+'.?')
if sci1[0] eq '' then begin
   if (strupcase(sc) eq 'A') then $
    	    sccattic=getenv_slash('scia')+'older/'+strmid(yyyymmdd,2,6)+'/' else $
    	    sccattic=getenv_slash('scib')+'older/'+strmid(yyyymmdd,2,6)+'/'
   sci1=file_search(sccattic+'*.??'+td)
   sci2=file_search(sccattic+'*.??'+td+'.?')
endif
IF sci2[0] NE '' THEN scifiles0=[sci1,sci2] ELSE scifiles0=sci1
break_file,scifiles0,v1,path,roots,sfxs
scifiles=roots+sfxs


if keyword_set(noattic) then begin
  ;--Check for files that are not in FITS directory but in DATABASE
  ;remove header only files from db list when no attic files are available
  tmp=where(strpos(imgdb.filename,'_h') eq -1)
  imgdb=imgdb(tmp)
  notfits =rest_mask(imgdb.filename,find_common(ftsfiles,imgdb.filename))
  if keyword_set(removeall) then begin
    nindb=n_elements(imgdb.filename)
    notfits=indgen(nindb)
  ENDIF
endif else notfits=[-1]

;--Check for files that are not in database but in FITS directory
t=find_common(imgdb.filename,ftsfiles)
if t(0) ne -1 then $
    	notindb =rest_mask(ftsfiles,find_common(imgdb.filename,ftsfiles)) else $
	   notindb=where(ftsfiles ne '')

if ~keyword_set(noattic) then begin
  ;--Check for files that are not in database but in sci (attic) directory
  t=find_common(imgdb.fileorig,scifiles)
  if t(0) ne -1 then  notindb2=rest_mask(scifiles,find_common(imgdb.fileorig,scifiles)) else $
                notindb2=where(scifiles ne '')
  ;--Check for files that are not in sci (attic) directory but in database 
  t=find_common(scifiles,imgdb.fileorig)
  if t(0) ne -1 then  notinattc=rest_mask(imgdb.fileorig,find_common(scifiles,imgdb.fileorig)) else $
                notinattc=where(imgdb.fileorig ne '')
  if keyword_set(removeall) then begin
    nindb=n_elements(imgdb.fileorig)
    notinattc=indgen(nindb)
  ENDIF
endif else notinattc=[-1]

close,1 ; ???
fileopen=0

;--Remove rows from database that are not in SCI / attic directory

natc=n_elements(notinattc)
IF notinattc[0] GE 0 THEN $
for i=0,natc-1 do begin
    n=notinattc[i]
    if ~keyword_set(noscript) then begin
    	openw,ludb,dbfile,width=200,/append
    	dbfstat=fstat(ludb)
    	IF dbfstat.size EQ 0 THEN BEGIN
    	; only at beginning of each script file
    	    printf,ludb,'use secchi_flight;'
    	    printf,ludb,'start transaction;'
    	    starttime=utc2str(ustarttime,/ecs)
    	    printf,ludb,'# starting dbfile at '+starttime
    	ENDIF
    	printf,ludb,'delete from img_seb_hdr_ext where fileorig = "',imgdb(n).fileorig,'" and date_obs = "',strmid(imgdb(n).DATE_OBS,0,10), 'T', strmid(imgdb(n).DATE_OBS,11,8),'";'
    	printf,ludb,'delete from img_hist_cmnt where date_obs = "',strmid(imgdb(n).DATE_OBS,0,10),'T',strmid(imgdb(n).DATE_OBS,11,8),'" and filename = "', imgdb(n).filename,'";'
    	printf,ludb,'delete from img_seb_hdr where fileorig = "',imgdb(n).fileorig,'" and date_obs = "',strmid(imgdb(n).DATE_OBS,0,10), 'T', strmid(imgdb(n).DATE_OBS,11,8),'" and filename = "',imgdb(n).filename,'";' 
    	close,ludb
    endif
    openw,lulog,logfile,width=200,/append
    IF dbfstat.size EQ 0 THEN BEGIN
    ; only at beginning of each script file
    	printf,lulog,'Number of images in database '+string(n_Elements(imgdb.filename))
        printf,lulog,'Number of images in sci directory '+string(n_elements(scifiles))
	printf,lulog,yyyymmdd+' '+tel+' removing from database '+trim(natc)+' not in attic sci dir'
    ENDIF
    printf,lulog,imgdb(n).filename+'  fileorig='+imgdb(n).fileorig+'  is not in attic sci directory'
    close,lulog
endfor

natc=n_elements(notfits)
IF notfits[0] GE 0 THEN $
for i=0,natc-1 do begin
    n=notfits[i]
    if ~keyword_set(noscript) then begin
    	openw,ludb,dbfile,width=200,/append
    	dbfstat=fstat(ludb)
    	IF dbfstat.size EQ 0 THEN BEGIN
    	; only at beginning of each script file
    	    printf,ludb,'use secchi_flight;'
    	    printf,ludb,'start transaction;'
    	    starttime=utc2str(ustarttime,/ecs)
    	    printf,ludb,'# starting dbfile at '+starttime
    	ENDIF
    	printf,ludb,'delete from img_seb_hdr_ext where fileorig = "',imgdb(n).fileorig,'" and date_obs = "',strmid(imgdb(n).DATE_OBS,0,10), 'T', strmid(imgdb(n).DATE_OBS,11,8),'";'
    	printf,ludb,'delete from img_hist_cmnt where date_obs = "',strmid(imgdb(n).DATE_OBS,0,10),'T',strmid(imgdb(n).DATE_OBS,11,8),'" and filename = "', imgdb(n).filename,'";'
    	printf,ludb,'delete from img_seb_hdr where fileorig = "',imgdb(n).fileorig,'" and date_obs = "',strmid(imgdb(n).DATE_OBS,0,10), 'T', strmid(imgdb(n).DATE_OBS,11,8),'" and filename = "',imgdb(n).filename,'";' 
    	close,ludb
    endif
    openw,lulog,logfile,width=200,/append
    IF dbfstat.size EQ 0 THEN BEGIN
    ; only at beginning of each script file
    	printf,lulog,'Number of images in database '+string(n_Elements(imgdb.filename))
        printf,lulog,'Number of images in fits directory '+string(n_elements(ftsfiles))
	printf,lulog,yyyymmdd+' '+tel+' removing from database '+trim(natc)+' not in fits dir'
    ENDIF
    printf,lulog,imgdb(n).filename+'  fileorig='+imgdb(n).fileorig+'  is not in fits directory'
    close,lulog
endfor

;--Add rows to database that are in FITS directory but not in database
ndb=n_elements(notindb)
fileopen2=0
IF notindb[0] GE 0 THEN $
for j=0,ndb-1 do begin
    n=notindb[j]
;  if (imgcheck(0) eq -1)then begin
      openw,lulog,logfile,width=200,/append
      dbfstat=fstat(lulog)
      IF dbfstat.size EQ 0 THEN BEGIN
	; only at beginning of each log file
            printf,lulog,'Number of images in database '+string(n_Elements(imgdb.filename))
            printf,lulog,'Number of images in FITS directory '+string(n_elements(ftsfiles))
      ENDIF
    IF j EQ 0 THEN BEGIN
    	print,yyyymmdd,' ',tel,' adding to database ',trim(n_elements(ftsfiles)-n_Elements(imgdb.filename)),' in attic sci dir'
    	printf,lulog,yyyymmdd+' '+tel+' adding to database '+trim(n_elements(ftsfiles)-n_Elements(imgdb.filename))+' in FITS dir'
    ENDIF
;add image to script files
    if ~keyword_set(noscript) and ~keyword_set(noattic) then begin
	openw,ludb,dbfile,width=200,/append
	dbfstat=fstat(ludb)
	IF dbfstat.size EQ 0 THEN BEGIN
	; only at beginning of each script file
	    printf,ludb,'use secchi_flight;'
	    printf,ludb,'start transaction;'
	    starttime=utc2str(ustarttime,/ecs)
	    printf,ludb,'# starting dbfile at '+starttime
	    close,ludb
	ENDIF
    	close,ludb
       img=sccreadfits(ftsfiles0[n],scch,/nodata)
       printf,lulog,scch.filename+' pol='+trim(scch.polar)+' is not in database.'
        img_dir='img'
    	case strmid(scch.filename,16,1) of
    	    'k' : IF (scch.detector NE 'HI1') AND (scch.detector NE 'HI2') THEN img_dir='cal'   ; DARK
      	    'e' : img_dir='cal'   ; LED
      	    'c' : img_dir='cal'   ; CONT
    	 else : 
    	endcase
    	if(trim(strlowcase(scch.detector)) eq 'cor1' or trim(strlowcase(scch.detector)) eq 'cor2')then begin
    	    case strmid(scch.filename,16,1) of
    	    	'n' : img_dir='seq'
    	    	's' : img_dir='seq'
	    	else:
    	    endcase
    	endif
    ; SCIP door CLOSED or IN_TRANSIT go in cal; HI door CLOSED (only) go in cal
     	IF ((scch.doorstat NE 2) AND  (scch.detector NE 'HI1') AND (scch.detector NE 'HI2')) OR $
            ((scch.doorstat EQ 0) AND ((scch.detector EQ 'HI1') OR  (scch.detector EQ 'HI2'))) $
        THEN img_dir='cal' 

    	case trim(strlowcase(scch.detector)) of
    	    'hi1' : BEGIN
	    	 SUBdir=img_dir+'/hi_1'
	     END
    	    'hi2' : BEGIN
	    	    SUBdir=img_dir+'/hi_2'
	    END
    	    else  : SUBdir=img_dir+'/'+trim(strlowcase(scch.detector))
    	endcase

        unblock_image_hdrs,sccattic+scch.fileorig ;       spawn, unblock_cmd+' '+infile,/sh, unblockresult
	delflag=0
        write_db_scripts, ftsfiles0[n], /silent
    endif else printf,lulog,ftsfiles(n)+' is not in database'

    close,lulog
    ;endif
endfor	; add missing database rows
endif $	; valid database query result
ELSE BEGIN
    print,' ##### No rows in database for ',cmnd
    wait,5
    openw,lulog,logfile,width=200,/append
    printf,lulog,' ##### No rows in database for obsrvtry ='+aorb+' and date_cmd between "'+time1+'" and "'+time2+'" and detector ='+detect+nax
    close,lulog
ENDELSE

endfor	; 1 day

end