#!/usr/bin/python

#
# convertoutput.py
#
# Nov 3, 2009 -- Derrick Stolee
#
# Take the current list of output files and create a new set of 
# jobs.  To prevent repeating the job creation, files of the form
# out.* are moved to analyzed.*
#

import os

folder = "./"
jcount = 0
newjcount = 0
scount = 0
pcount = 0
ccount = 0
job = 0

import sys

#import folder
for i in range(1,len(sys.argv)):
	arg = sys.argv[i];
	if arg.startswith('-f'):
		folder = sys.argv[i+1];

print "Compacting folder " + str(folder)

os.system('touch ' + folder + 'allstats.txt')
statfile = open(folder + 'allstats.txt', 'r')
stats = { }
stattypes = { }

for line in statfile.xreadlines():
	if line[0] == 'T':
			# We're talking statistics!
		if line[2:5] == 'SUM':
			var, val = line[6:].split(' ')
			stats[var] = 0
			stattypes[var] = 'SUM'
			stats[var] += float(val)
		if line[2:5] == 'MAX':
			var, val = line[6:].split(' ')
			stats[var] = -1
			stattypes[var] = 'MAX'
			if float(val) > stats[var]:
				stats[var] = float(val) 

statfile.close()


os.system('ls ' + folder + ' | grep stage > ' + folder + 'liststagefile.txt')

stagefile = open(folder+'liststagefile.txt')

stagenum = 0
for line in stagefile.xreadlines():
	stagenum = stagenum + 1

os.system('rm ' + folder + 'liststagefile.txt')

jobfile = open(folder + 'alljobs.txt','w')
outjobfile = open(folder + 'newjobs.txt', 'w')
solfile = open(folder + 'allsols.txt','a')


# find the current job number, if there exist some already
os.system('ls ' + folder + ' | grep out > ' + folder + 'filelist.txt');
os.system('cat ' + folder + 'in.* > ' + folder + 'allin.txt')
os.system('cat ' + folder + 'err.* >> ' + folder + 'allerr.txt')

outfiles = open(folder + 'filelist.txt')

ocount = 0;
job = 0;
numfileschecked = 0;
for filename in outfiles.xreadlines():
	file = open(folder + filename.strip())
	
	numfileschecked += 1;
	if (numfileschecked % 100) == 0:
		print "\tChecked",numfileschecked,"files...";
	
	numlines = 0
	for f in file.xreadlines():
		numlines = numlines + 1
		line = f.strip()
		linedata = line.split(" ")	
		if len(line) == 0:
			continue
			
		# open this file out.*
		if line[0] == 'J':
			jobfile.write(line + "\n")
			newjcount = newjcount + 1
			jcount = jcount + 1
		elif line[0] == 'P':
			jobfile.write(line + "\n")
			pcount = pcount + 1
			jcount = jcount + 1
		elif line[0] == 'S':
			solfile.write(line + "\n")
			scount = scount + 1
		elif line[0] == 'C':
			ccount = ccount + 1
		elif line[0] == 'T':
			# We're talking statistics!
			if line[2:5] == 'SUM':
				if len(line[6:].split(' ')) < 2:
					print "Problem with stat in " + filename.strip() + ": " + line;
				else:
					var, val = line[6:].split(' ')
					if not var in stats.keys():
						stats[var] = 0
						stattypes[var] = 'SUM'	
					stats[var] += float(val)
			if line[2:5] == 'MAX':
				if len(line[6:].split(' ')) < 2:
					print "Problem with stat in " + filename.strip() + ": " + line;
				else:
					var, val = line[6:].split(' ')
					if not var in stats.keys():
						stats[var] = -1
						stattypes[var] = 'MAX'
					if float(val) > stats[var]:
						stats[var] = float(val) 	
		elif len(line) > 0:
			# this must be data from a solution!
			solfile.write(line  + "\n")
	
	file.close()
	
	if numlines == 0:
		# failed job!
		file = open(folder + "in" + filename[3:len(filename)-1])
		for f in file.xreadlines():
			ocount = ocount + 1
			jobfile.write(f)
		file.close()
outfiles.close()	

os.system('rm ' + folder + 'out.*')
os.system('rm ' + folder + 'in.*')
os.system('rm ' + folder + 'err.*')
	
solfile.close()


# need to fill the job list with the back-jobs
bcount = 0

os.system('touch ' + folder + 'backjobs.txt')

outjobs = open(folder + 'backjobs.txt','r')
for line in outjobs.xreadlines():
	jobfile.write(line)
	bcount = bcount + 1


# done writing jobs
jobfile.close()
outjobs.close()

# remove the old backjobs
os.system('rm -rf ' + folder + 'backjobs.txt')

print "Found " + str(jcount + ocount) + " jobs (" + str(pcount) + " partial, " + str(ccount) + " complete, " + str(ocount) + " failed)"
print "Found " + str(newjcount) + " NEW jobs "
print "Found " + str(bcount) + " back-jobs "
print "Found " + str(scount) + " solutions!"

# combine all of the statistics! 
statfile = open(folder + 'allstats.txt', 'w')
for k in stats.keys():
        statfile.write('T ' + stattypes[k] + ' ' + k + ' ' + str(stats[k]) + '\n')
statfile.close()

# save data for new stages!
os.system('mkdir ' + folder + 'stage' + str(stagenum) )
os.system('mv ' + folder + 'log* ' + folder + 'stage' + str(stagenum) + '/')
os.system('mv ' + folder + 'condorsubmit.sub ' + folder + 'stage' + str(stagenum) + '/')
os.system('cp ' + folder + '*.txt ' + folder + 'stage' + str(stagenum) + '/')




