// (C)2007 Attila Kovacs <attila@submm.caltech.edu>

import java.io.*;
import java.net.*;
import java.util.*;

public class MiniCRUSH {
	static String version = "1.02-2";
	static String projectID = "T-79.F-0002-2007";
	static String workPath = ".";

	TauInterpolator tauInterpolator;
	CalibrationTable calibrationTable;
	
	SkyMap map;
	LabocaScan[][] subscan;
	int scans;

	double beamFWHM = 18.0 * Unit.arcsec;
	double resolution = 6.0 * Unit.arcsec;
	double sourceSize = Double.NaN;
	double extendedFWHM = Double.NaN;
	boolean beamMap = false;
	boolean chopped = false;

	String instrument = "laboca";
	
	Hashtable<String,String> options = new Hashtable<String,String>();
	
	Vector<String> extendedOptions = new Vector<String>();
	Vector<String> blacklist = new Vector<String>();
	Hashtable<String, Vector<String>> iterationOptions = new Hashtable<String, Vector<String>>();
	
	Vector<String> ordering = new Vector<String>();
	Vector<ReductionThread> threads = new Vector<ReductionThread>();
	
	int useCPUs = 1;
	
	public static void main(String[] args) {
		info();
		
		if(args.length == 0) {
			usage();
			System.exit(0);			
		}
		
		if(args[0].equalsIgnoreCase("-help")) {
			help();
			System.exit(0);	
		}
		
		MiniCRUSH reducer = init(args);
		
		try { reducer.reduce(); }
		catch(InterruptedException e) {}
	}

	public MiniCRUSH(String instrument) {
		checkForUpdates();
		//getMessages();
		
		this.instrument = instrument.toLowerCase();
		
		options.put("decorrelate", "");
		options.put("offsets", "");
		options.put("sourcemap", "");
		
		if(instrument.equalsIgnoreCase("laboca")) {
			scanTemplate = new LabocaScan();
		}
		else if(instrument.equalsIgnoreCase("saboca")) {
			scanTemplate = new SabocaScan();
		}
		else if(instrument.equalsIgnoreCase("aszca")) {
			scanTemplate = new APEXSZScan();
		}
		else {
			System.err.println("Warning! Unknown instrument " + instrument);
			System.exit(1);
		}
		
		System.err.println("Instrument is " + instrument.toUpperCase());
	}
	

	public void readConfigFile(String fileName) throws IOException {
		File configFile = new File(fileName);
		if(configFile.exists()) {
			System.err.println("Reading configuration from " + fileName);
			
			BufferedReader in = new BufferedReader(new InputStreamReader(new FileInputStream(configFile)));
			String line = null;
			while((line = in.readLine()) != null) if(line.length() > 0) if(line.charAt(0) != '#') option(line);
		}
		else throw new FileNotFoundException(fileName);
	}
	
	public void option(String line) {
		StringTokenizer tokens = new StringTokenizer(line, ":= \t");
		String key = tokens.nextToken().toLowerCase();
		
		if(blacklist.contains(key)) return;
		
		// The subvalue is the rest after the key and the first set of separators...
		String value = "";
		if(tokens.hasMoreTokens()) {
			String rest = line.substring(key.length());
			value = rest.substring(rest.indexOf(tokens.nextToken().charAt(0)));
		}
		
		//System.err.println(key + " = " + value);
		
		if(key.equalsIgnoreCase("config")) readConfig(Util.getSystemString(value));
		else if(key.equalsIgnoreCase("read")) read(Util.getSystemString(value));
		else if(key.startsWith("iteration")) {
			if(key.equals("iterations")) {
				if(value.equalsIgnoreCase("clear")) iterationOptions.clear();
				else for(String spec : iterationOptions.keySet()) iterationOptions.get(spec).add(value);
			}
			else {
				String spec = key.substring(key.indexOf('.')+1);
				if(!iterationOptions.containsKey(spec)) iterationOptions.put(spec, new Vector<String>());
		
				if(value.equalsIgnoreCase("clear")) iterationOptions.get(spec).clear();
				else iterationOptions.get(spec).add(value);
			}
		}
		else if(key.equalsIgnoreCase("tau")) {
			try { tauInterpolator = new TauInterpolator(value); }
			catch(IOException e) { tauInterpolator = null; }
			options.put(key, value); 	
		}
		else if(key.equalsIgnoreCase("scale")) {
			try { calibrationTable = new CalibrationTable(value); }
			catch(IOException e) { calibrationTable = null; }
			options.put(key, value); 
		}
		else if(key.equalsIgnoreCase("forget")) {
			StringTokenizer list = new StringTokenizer(value, ",");
			while(list.hasMoreTokens()) options.remove(list.nextToken().toLowerCase());
		}
		else if(key.equalsIgnoreCase("blacklist")) {
			if(!blacklist.contains(value.toLowerCase())) {
				if(value.equalsIgnoreCase("clear")) blacklist.clear();
				else {
					StringTokenizer list = new StringTokenizer(value, ",");
					while(list.hasMoreTokens()) blacklist.add(list.nextToken().toLowerCase());
					option("forget=" + value);
				}
			}
		}
		else if(key.equalsIgnoreCase("whitelist")) {
			StringTokenizer list = new StringTokenizer(value, ",");
			while(list.hasMoreTokens()) {
				String optionKey = list.nextToken().toLowerCase();
				if(blacklist.contains(optionKey)) blacklist.remove(optionKey);
			}
		}
		else if(key.equalsIgnoreCase("extendedopt")) {
			if(value.equalsIgnoreCase("clear")) extendedOptions.clear();
			else extendedOptions.add(value);
		}
		// TODO...
		else if(key.equalsIgnoreCase("poll")) {
			Hashtable<String, String> configuration = getConfiguration();
			System.err.println();
			if(value == "") printConfig(System.out);
			else {
				StringTokenizer keys = new StringTokenizer(value, ", \t");
				while(keys.hasMoreTokens()) {
					String pollkey = keys.nextToken().toLowerCase();
					boolean found = false;
					for(String option : configuration.keySet()) {
						if(option.startsWith(pollkey)) {
							System.out.println("   " + option + "  " + configuration.get(option));
							found = true;
						}
					}
					if(!found) System.out.println("# option " + pollkey + " undefined.");
				}
			}
			System.err.println();
			System.exit(0);
		}
		else options.put(key, value);

		if(key.equals("bright")) readConfig("bright.cfg");
		else if(key.equals("deep")) readConfig("deep.cfg");	 
	}

	// First looks for configuration in ~/.minicrush/
	// then in the current directory.	
	public void readConfig(String fileName) {
		try { readConfigFile(System.getProperty("user.home") + File.separator + ".minicrush" + File.separator + fileName); }
		catch(IOException e) {
			try { readConfigFile(fileName); }
			catch(IOException e4) {	}
		}
		
		// read the instrument overriderrs (if any).
		try { readConfigFile(System.getProperty("user.home") + File.separator + ".minicrush" + File.separator + instrument + File.separator + fileName); }
		catch(IOException e) {
			try { readConfigFile(instrument + File.separator + fileName); }
			catch(IOException e4) { }
		}
		
	}

	private LabocaScan scanTemplate;
	private Vector<LabocaScan[]> scanBuffer = new Vector<LabocaScan[]>();

	public static MiniCRUSH init(String[] args) {
		MiniCRUSH reducer = new MiniCRUSH(args[0]);

		reducer.readConfig("default.cfg");
		reducer.readConfig("minicrush.cfg");		
		
		for(int i=1; i<args.length; i++) {
			if(args[i].charAt(0) == '-') {
				String option = args[i].substring(1);
				reducer.option(option);

				if(option.equals("bright")) reducer.readConfig("bright.cfg"); 
				else if(option.equals("faint")) reducer.readConfig("faint.cfg"); 
				else if(option.equals("deep")) reducer.readConfig("deep.cfg"); 
				else if(option.equals("extended")) reducer.readConfig("extended.cfg"); 
			}
			else reducer.read(args[i]);
		}
		
		reducer.validate();

		return reducer;
	}
	

	public void validate() {	
		if(options.containsKey("chopped")) chopped = true;
		if(options.containsKey("outpath")) workPath = Util.getSystemString(options.get("outpath"));
		if(options.containsKey("beam")) beamFWHM = Double.parseDouble(options.get("beam")) * Unit.arcsec;
		
		if(options.containsKey("maxhe3rms")) {
			double maxrms = 1e-3 * Double.parseDouble(options.get("maxhe3rms"));
			for(int i=0; i<scanBuffer.size(); ) {
				if(scanBuffer.get(i)[0].rmsHe3 > maxrms) {
					System.err.println("Scan " + scanBuffer.get(i)[0].serialNo + " temperature fluctuations exceed limit. Removing from dataset.");
					scanBuffer.remove(i);
				}
				else i++;
			}
		}
		
		if(scanBuffer.size() == 0) {
			System.err.println("No scans to reduce. Exiting.");
			System.exit(1);
		}
		
		if(options.containsKey("ordering")) {
			StringTokenizer entries = new StringTokenizer(options.get("ordering"), " :,;\t");
			ordering.clear();
			while(entries.hasMoreTokens()) { ordering.add(entries.nextToken()); } 
		}
	
		scans = scanBuffer.size();
		subscan = new LabocaScan[scans][];
		for(int i=0; i<scans; i++) subscan[i] = scanBuffer.get(i);
		
		
		useCPUs = Runtime.getRuntime().availableProcessors();
		if(options.containsKey("reservecpus")) useCPUs -= Integer.parseInt(options.get("reservecpus"));
		useCPUs = Math.max(useCPUs, 1);
		System.err.println("Will use up to " + useCPUs + " CPU(s).");
		
		
	}


	public void read(String scanID) {
		StringTokenizer list = new StringTokenizer(scanID, "; \t");
		StringTokenizer range = new StringTokenizer(scanID, "-");
		
		if(list.countTokens() > 1) while(list.hasMoreTokens()) read(list.nextToken());
		else if(range.countTokens() > 1) {
			int from = Integer.parseInt(range.nextToken());
			int to = Integer.parseInt(range.nextToken());
			for(int no = from; no <= to; no++) read(no + "");
		}
		else {	
			try {
				scanTemplate.scramble = options.containsKey("scramble");
				scanTemplate.isChopped = options.containsKey("chopped");
				if(options.containsKey("beam")) scanTemplate.beamFWHM = Double.parseDouble(options.get("beam")) * Unit.arcsec;	
				if(options.containsKey("jansky")) scanTemplate.janskyPerMicroV = Double.parseDouble(options.get("jansky"));
				if(options.containsKey("datapath"))
					LabocaScan.path = Util.getSystemString(options.get("datapath"));
				if(options.containsKey("project"))
					projectID = options.get("project");
				if(options.containsKey("he3")) {
					scanTemplate.useHe3Data = options.get("he3").equals("thermistor");
					scanTemplate.useBlindData = options.get("he3").equals("blinds");
					scanTemplate.calcHe3Gains = options.get("he3").equals("gains");
				}
				if(options.containsKey("taufactor")) LabocaFrame.skydipFactor = Double.parseDouble(options.get("taufactor")); 
				if(options.containsKey("hipass")) scanTemplate.hipassTimeScale = Double.parseDouble(options.get("hipass"));
				if(options.containsKey("vclip")) {
					StringTokenizer tokens = new StringTokenizer(options.get("vclip"), " \t,-");
					scanTemplate.minScanSpeed = Double.parseDouble(tokens.nextToken()) * Unit.arcsec/Unit.sec;
					if(tokens.hasMoreTokens()) scanTemplate.maxScanSpeed = Double.parseDouble(tokens.nextToken()) * Unit.arcsec/Unit.sec;
					if(options.containsKey("photometry")) {
						scanTemplate.minScanSpeed = 0.0;
					}
					if(options.containsKey("chopped")) {
						scanTemplate.minScanSpeed = 0.0;
						scanTemplate.maxScanSpeed = Double.NaN;
					}
 				}
				if(options.containsKey("aclip")) {
					scanTemplate.maxAcceleration = Double.parseDouble(options.get("aclip")) * Unit.arcsec/Unit.s2;
					if(options.containsKey("chopped")) {
						scanTemplate.maxAcceleration = Double.NaN;
					}
				}
				if(options.containsKey("rcp"))
					scanTemplate.rcpFile = options.get("rcp");
				if(options.containsKey("pixeldata")) if(!options.get("pixeldata").equalsIgnoreCase("write")) 
					scanTemplate.pixelDataFile = options.get("pixeldata");
				
				scanTemplate.flaggedPixels = new Vector<Integer>();	
				if(options.containsKey("flag")) {
					StringTokenizer pixelList = new StringTokenizer(options.get("flag"), ", \t");
					while(pixelList.hasMoreTokens()) scanTemplate.flaggedPixels.add(Integer.parseInt(pixelList.nextToken()));				
				}
			
				scanTemplate.uniformGains = options.containsKey("uniform");
				
				LabocaScan[] set = scanTemplate.read(this, Integer.parseInt(scanID), projectID, options.containsKey("compressed"));
						
				if(options.containsKey("photometry")) {
					for(int i=0; i<set.length; i++) set[i].blankSource(new Vector2D(), beamFWHM, beamFWHM/5.0);
				}
				if(!options.containsKey("chopped")) set = new LabocaScan[] { LabocaScan.unite(set) };
				
				if(options.containsKey("center")) {			
					StringTokenizer tokens = new StringTokenizer(options.get("center"),",");
					double dAZ = Double.parseDouble(tokens.nextToken());
					double dEL = Double.parseDouble(tokens.nextToken());
					System.err.print(" Adjusting pointing by " + dAZ/Unit.arcsec + "," + dEL/Unit.arcsec + " arcsec.");
					for(int k=0; k<set.length; k++) 
						set[k].pointingAt(new Vector2D(dAZ*Unit.arcsec, dEL*Unit.arcsec));
				}
				if(options.containsKey("shift")) {
					double dt = Double.parseDouble(options.get("shift")) * Unit.s;
					System.err.print(" Shifting by " + dt/Unit.s + " sec.");

					for(int k=0; k<set.length; k++) set[k].shift(dt);
					System.err.println();
				}
				if(options.containsKey("downsample")) {
					int factor = Integer.parseInt(options.get("downsample"));
					for(int k=0; k<set.length; k++) {
						set[k].downsample(factor);
						set[k].trim();
					}
				}
				if(options.containsKey("frames")) {
					StringTokenizer tokens = new StringTokenizer(options.get("frames"), "-");
					int from = Integer.parseInt(tokens.nextToken());
					int to = Integer.parseInt(tokens.nextToken());
					for(int k=0; k<set.length; k++) {
						LabocaFrame[] frame = new LabocaFrame[to-from];
						for(int t=from, tot=0; t<to; t++,tot++) frame[tot] = set[k].frame[t];
						set[k].frames = frame.length;
						set[k].frame = frame;
					}
				}	
				if(options.containsKey("tau")) {
					for(int k=0; k<set.length; k++) {
						try { set[k].setZenithTau(tauInterpolator == null ? 
								Double.parseDouble(options.get("tau")) 
								: tauInterpolator.getValue(set[k].getMJD()));
						}
						catch(ArrayIndexOutOfBoundsException e) {
							System.err.println("     WARNING! " + e.getMessage());
							set[k].setZenithTau(0.0);							
						}
					}
				}
				if(options.containsKey("scale")) {
					for(int k=0; k<set.length; k++) {
						double cal = 1.0;
						try { cal = calibrationTable == null ? 
								Double.parseDouble(options.get("scale")) 
								: calibrationTable.getValue(set[k].getMJD());
						}
						catch(ArrayIndexOutOfBoundsException e) {
							System.err.println("     WARNING! " + e.getMessage());
						}
						System.err.println("   Applying relative calibration factor " + Util.f3.format(cal));
						set[k].gain = 1.0 / cal;
					}
						
				}
				if(options.containsKey("jacknife")) {
					// Randomly invert the signs of half the scans...
					if(Math.random() < 0.5) for(int k=0; k<set.length; k++) set[k].gain = -1.0;					
				}
				
				scanBuffer.add(set);
				
				System.gc();
				
				System.err.println();
			}
			catch(Exception e) { 
				System.err.println("ERROR! " + e.getMessage());
				e.printStackTrace();
			}
		}
	}

	public void scanSetup() {		
		for(int i=0; i<scans; i++) for(int k=0; k<subscan[i].length; k++) subscan[i][k].createStorage();
		
		System.err.println();
	}
	

	public String getShorthand(Vector<String> list) {
		String shorthand = "";
		for(String spec : list) {
			if(shorthand.length() > 0) shorthand += "|";
			shorthand += spec; 
		}
		return shorthand;
	}
	
	public Hashtable<String, String> getConfiguration() {
		Hashtable<String, String> configuration = new Hashtable<String, String>();
		for(String key : options.keySet()) configuration.put(key, options.get(key));
		for(String key : iterationOptions.keySet()) configuration.put("iteration." + key, getShorthand(iterationOptions.get(key)));
		configuration.put("extendedopt", getShorthand(extendedOptions));
		configuration.put("blacklist", getShorthand(blacklist));
		return configuration;
	}
	
	public void reduce() throws InterruptedException {
		
		if(options.containsKey("extended")) for(String spec : extendedOptions) {
			StringTokenizer list = new StringTokenizer(spec, "|");
			while(list.hasMoreTokens()) option(list.nextToken());
		}
		
		int rounds = 0;
		
		if(options.containsKey("bright")) System.out.println("Bright source reduction.");
		else if(options.containsKey("faint")) System.out.println("Faint source reduction.");
		else if(options.containsKey("deep")) System.out.println("Deep source reduction.");
		else System.out.println("Default reduction.");
		
		
		if(options.containsKey("faint") || options.containsKey("deep")) {
			// extended overrides sourcesize!
			if(options.containsKey("extended")) options.remove("sourcesize");
			else if(options.containsKey("sourcesize")) {
				sourceSize = Double.parseDouble(options.get("sourcesize")) * Unit.arcsec;
				if(sourceSize < beamFWHM) sourceSize = beamFWHM;
			}
			else sourceSize = beamFWHM;
		}

		if(!Double.isNaN(sourceSize)) System.out.println("Assuming ca. " + (sourceSize/Unit.arcsec) + "\" source(s).");
		else if(options.containsKey("extended")) System.out.println("Assuming extended source(s).");
		else  System.out.println("Assuming compact source(s).");
		
		if(options.containsKey("rcpgains")) System.out.println("Will use source gains from RCP.");
		
		System.out.println();
		
		init();	
		
		
		if(options.containsKey("rounds")) rounds = Integer.parseInt(options.get("rounds"));
		
		if(iterationOptions.containsKey("last")) {
			iterationOptions.put("last-0", iterationOptions.get("last"));
			iterationOptions.remove("last");
		}
		
		for(int k=0; k<rounds; k++) {
			if(iterationOptions.containsKey("last-" + k)) {
				if(!iterationOptions.containsKey((rounds-k) + "")) 
					iterationOptions.put((rounds-k) + "", new Vector<String>());
				iterationOptions.get((rounds-k) + "").addAll(iterationOptions.get("last-" + k));
				iterationOptions.remove("last-" + k);
			}
		}       	

		for(int iteration=1; iteration<=rounds; iteration++) {

			System.err.println("  Round " + iteration + ": ");	

			if(iterationOptions.containsKey(iteration + ""))
				for(String spec : iterationOptions.get(iteration + "")) {
					StringTokenizer list = new StringTokenizer(spec, "|");
					while(list.hasMoreTokens()) option(list.nextToken());
				}

			iterate(ordering);

			System.err.println();
		}

		System.err.println();

		if(options.containsKey("photometry")) {
			System.err.println("Obtaining photometric data:");
			System.err.println();
			double toJy = 1.0e6 * Double.parseDouble(options.get("jansky"));
			WeightedPoint flux = new WeightedPoint();
			for(int i=0; i<scans; i++) for(int k=0; k<subscan[i].length; k++) {
				WeightedPoint exposureFlux = subscan[i][k].getPhotometry();
				exposureFlux.scale(toJy);
				System.out.print("\t[" + subscan[i][k].serialNo + "|" + subscan[i][k].subscanNo + "] ");
				System.out.println(exposureFlux.toString(Util.e3));
				flux.average(exposureFlux);
			}
			System.err.println();
			System.err.println("\tFINAL: " + flux.toString(Util.e3) + " Jy/beam");
		}
		
		if(options.containsKey("chopped")) {
			reduceChopped();
		}
		
		if(map != null) {	
			// Use imageWeights unless extended structures would argue against it...
			if(!options.containsKey("extended") || options.containsKey("deep")) map.robustWeight(true);
		
			try { 
				if(options.containsKey("name")) map.fileName = workPath + File.separator + Util.getSystemString(options.get("name"));
				else {
					map.fileName = workPath + File.separator + map.sourceName + "." + subscan[0][0].serialNo;
					if(scans > 1) map.fileName += "-" + subscan[scans-1][0].serialNo;
					map.fileName += ".fits";
				}
				if(options.containsKey("unit")) map.setUnit(options.get("unit"));
				
				map.configuration = getConfiguration();
				map.info();
				map.write(); 
			}
			catch(Exception e) { e.printStackTrace(); }
		}

		if(options.containsKey("pixeldata")) if(options.get("pixeldata").equalsIgnoreCase("write")) for(int i=0; i<scans; i++) for(int k=0; k<subscan[i].length; k++) {
			String fileName = workPath + File.separator + "pixel-" + subscan[i][k].serialNo + ".dat";
			try { LabocaPixel.writePixelData(subscan[i][k].channel, fileName); }
			catch(Exception e) { e.printStackTrace(); }
		}

		if(options.containsKey("noisestats")) addNoiseStats();

		if(options.containsKey("covars")) {
			for(int i=0; i<scans; i++) for(int k=0; k<subscan[i].length; k++) {
				try {
					double[][] covar = subscan[i][k].getCovariance();
					subscan[i][k].writeCovariance(workPath + File.separator + "covar", subscan[i][k].getFullCovariance(covar));
					subscan[i][k].writeCovariance(workPath + File.separator + "covar-box", LabocaScan.getGroupCovariance(subscan[i][k].getChannelGroup("boxes"), covar));
					subscan[i][k].writeCovariance(workPath + File.separator + "covar-cable", LabocaScan.getGroupCovariance(subscan[i][k].getChannelGroup("cables"), covar));
					subscan[i][k].writeCovariance(workPath + File.separator + "covar-amp", LabocaScan.getGroupCovariance(subscan[i][k].getChannelGroup("amps"), covar));
					subscan[i][k].writeCovariance(workPath + File.separator + "covar-wafer", LabocaScan.getGroupCovariance(subscan[i][k].getChannelGroup("wafers"), covar));
					subscan[i][k].writeCovariance(workPath + File.separator + "covar-region", LabocaScan.getGroupCovariance(subscan[i][k].getChannelGroup("regions"), covar));
				}
				catch(Exception e) { e.printStackTrace(); }
			}
		}

		if(options.containsKey("log")) {
			try { logScans(); }
			catch(Exception e) { e.printStackTrace(); }
		}

		// Remove the intermediate image file...
		File intermediate = new File("intermediate.fits");
		if(intermediate.exists()) intermediate.delete();

			
		if(options.containsKey("ascii")) {
			for(int i=0; i<scans; i++) for(int k=0; k<subscan[i].length; k++) {
				try { subscan[i][k].printASCIITimeStream(); }
				catch(Exception e) { e.printStackTrace(); }
			}
		}
		
		if(options.containsKey("spectrum")) {
			String windowName = "Hamming";
			int windowSize = 512;
			
			StringTokenizer tokens = new StringTokenizer(options.get("spectrum"));
			
			while(tokens.hasMoreTokens()) {
				String value = tokens.nextToken();
				try { windowSize = Integer.parseInt(value); }
				catch(NumberFormatException e) { windowName = value; }
			}
			
			for(int i=0; i<scans; i++) for(int k=0; k<subscan[i].length; k++) {
				try { subscan[i][k].writeSpectra(windowName, windowSize); }
				catch(Exception e) { e.printStackTrace(); }
			}
		}
	}
	
	
	public void reduceChopped() {				
		WeightedPoint flux = new WeightedPoint();
	
		for(int i=0; i<scans; i++) {
			WeightedPoint leftSource = new WeightedPoint();
			WeightedPoint rightSource = new WeightedPoint();

			for(int k=0; k<subscan[i].length; k++) {
				for(int c=0; c<subscan[i][k].channels; c++) {
					LabocaPixel pixel = subscan[i][k].channel[c];
					if((pixel.flag & LabocaPixel.CHOPPED_LEFT_BEAM) != 0) 
						leftSource.average(new ChoppingData(subscan[i][k], pixel).getLeftValue()); 
					else if((pixel.flag & LabocaPixel.CHOPPED_RIGHT_BEAM) != 0) 
						rightSource.average(new ChoppingData(subscan[i][k], pixel).getRightValue()); 
				}
			}
			
			leftSource.add(rightSource);
			leftSource.scale(0.5);
			
			flux.average(leftSource);
			
			System.err.print("  [" + subscan[i][0].serialNo + "] ");
			
			if(options.containsKey("jansky")) {
				leftSource.scale(1e6 * Double.parseDouble(options.get("jansky")));
				System.err.println(leftSource.toString(Util.e3) + " Jy/beam");
			}		
			else System.err.println(leftSource.toString(Util.e3) + "uV");
		}

		flux.scale(1.0e6);
		
		System.err.println();
		System.err.print("  Total: ");
		
		if(options.containsKey("jansky")) {
			flux.scale(Double.parseDouble(options.get("jansky")));
			System.err.println(flux.toString(Util.e3) + " Jy/beam");
		}		
		else System.err.println(flux.toString(Util.e3) + "uV");
		
	}


	double[][] base;

	
	public synchronized void iterate(Vector<String> modelOrdering) throws InterruptedException {
		while(!queue.isEmpty()) wait();
		
		System.err.println();
			
		boolean modelSource = options.containsKey("sourcemap") && modelOrdering.contains("sourcemap") && !(options.containsKey("photometry") || options.containsKey("chopped"));
		
		if(options.containsKey("photometry") || options.containsKey("chopped")) {
			options.put("drifts", options.containsKey("stability") ? options.get("stability") : "30.0");
			options.remove("sourcemap");
			options.remove("whiten");
		}
			
		threads.clear();	
		
		for(int i=0; i<useCPUs; i++) {
			ReductionThread thread = new ReductionThread(this);
			thread.setOrdering(modelOrdering);
			threads.add(thread);
		}
		
		
		for(int i=0; i<scans; i++) threads.get(i % useCPUs).addScan(subscan[i]);
		
		for(int i=1; i<useCPUs && i<threads.size();) {
			if(threads.get(i).scans.size() == 0) threads.remove(i);
			else i++;
		}
		
		// Clear the source map only a new ones is to be made... 
		if(modelSource) {
			base = (double[][]) map.imageCopy(map.signal);
			map.reset();
		}
		else base = null;
			
		for(ReductionThread thread : threads) thread.start();
	
		summarize();
		
		if(modelSource) processMap(base);
		
		if(options.containsKey("whiten")) if(options.get("whiten").equalsIgnoreCase("once")) options.remove("whiten");
		
		System.gc();
	}
	
	
	
	Vector<LabocaScan> queue = new Vector<LabocaScan>();
	
	
	public synchronized void summarize() throws InterruptedException {
		// Go in order.
		// If next one dissappears from queue then print summary
		// else wait();
		
		for(int i=0; i<scans; i++) {
			for(int k=0; k<subscan[i].length; k++) {
				while(queue.contains(subscan[i][k])) wait();
				summarize(subscan[i][k]);
				notifyAll();
			}
			//System.err.println();
		}
		System.err.println();
		
	}
	
	public void printPipeline() {
		System.err.println("   Pipeline is: ");
	}
	
	public synchronized void checkout(LabocaScan scan) { 
		//scan.calcAverageFiltering();
		//scan.comments += "[" + Util.e2.format(scan.nefd / scan.averageFiltering) + "]"; 
		
		queue.remove(scan);
		notifyAll(); 
	}
	
	// TODO move map processing here
	public void processMap(double[][] base) {
		System.err.print("   [Source] ");
		
		map.normalize();
		map.calc();
		
	
		double blankingLevel = Double.NaN;
		if(options.containsKey("blank")) blankingLevel = Double.parseDouble(options.get("blank"));
				
		if(options.containsKey("scanmapdespike")) System.err.print("{despike} ");
	
		if(options.containsKey("scanweighting"))
			for(int i=0; i<scans; i++) System.err.print("{" + Util.f2.format(subscan[i][0].weight) + "} ");
	
		
		if(options.containsKey("filter") && sourceSize > 0.0) {
			System.err.print("(filter) ");
			StringTokenizer tokens = new StringTokenizer(options.get("filter"), ", \t");
			String mode = tokens.nextToken();
			double filterScale = 5.0 * sourceSize;
			double filterBlanking = tokens.hasMoreTokens() ? Double.parseDouble(tokens.nextToken()) : blankingLevel;
			if(mode.equalsIgnoreCase("fft")) map.fftFilterAbove(filterScale, filterBlanking);
			else map.filterAbove(filterScale, filterBlanking);
			map.extFilterFWHM = filterScale;
			map.filterBlanking = filterBlanking;
		}
		else {
			map.extFilterFWHM = Double.NaN;
			map.filterBlanking = Double.NaN;
		}
	
		if(options.containsKey("smooth")) {
			System.err.print("(smooth) ");
			map.convolveTo(Double.parseDouble(options.get("smooth")) * Unit.arcsec);
		}
				
		// Noise and exposure clip after smoothing for evened-out coverage...
		// Eposure clip
		if(options.containsKey("exposureclip")) {
			System.err.print("(exposureclip) ");
			map.exposureClip(Double.parseDouble(options.get("exposureclip")));		
		}

		// Noise clip
		if(options.containsKey("noiseclip")) {
			System.err.print("(noiseclip) ");
			map.rmsClip(Double.parseDouble(options.get("noiseclip")));		
		}

		if(options.containsKey("clip")) {	
			double clipLevel = Double.parseDouble(options.get("clip"));
			System.err.print("(clip:" + clipLevel + ") ");
			map.s2nClip(clipLevel);
			map.clippingS2N = clipLevel;
		}

		map.sanitize();
		
		String sourceEstimator = options.get("sourcemap");
		if(sourceEstimator.equalsIgnoreCase("mem")) {
			System.err.print("(MEM) ");
			double lambda = options.containsKey("mem-lambda") ? Double.parseDouble(options.get("mem-lambda")) : 0.1;
			map.MEM(lambda, true);
		}
		
		if(options.containsKey("intermediatemaps")) {
			map.fileName = "intermediate.fits";
			try { map.write(); }
			catch(Exception e) { e.printStackTrace(); }
		}
		
		// TODO make source sync multi-threaded...
		// Coupled with blanking...
		if(!options.get("sourcemap").contains("nosync")) {
			if(options.containsKey("blank")) System.err.print("(blank:" + blankingLevel + ") ");
			System.err.print("(sync) ");
			try { syncSource(); }
			catch(InterruptedException e) { e.printStackTrace(); }
		}
		
		map.generation++;
		
	}
	
	Vector<SourceSyncThread> syncThreads = new Vector<SourceSyncThread>();
	
	
	
	class SourceSyncThread extends Thread {
		Vector<LabocaScan[]> scans = new Vector<LabocaScan[]>();
		MiniCRUSH crush;
		
		public SourceSyncThread(MiniCRUSH crush) { this.crush = crush; }
		
		public void addScan(LabocaScan[] scan) { 
			scans.add(scan); 
			for(int k=0; k<scan.length; k++) crush.queue.add(scan[k]);
		}
		
		public void run() {
			double blankingLevel = Double.NaN;
			if(crush.options.containsKey("blank")) blankingLevel = Double.parseDouble(crush.options.get("blank"));
			
			for(LabocaScan[] scan : scans) for(int k=0; k<scan.length; k++) {
				scan[k].removeSource(map, base, blankingLevel);				
				crush.checkout(scan[k]);
			}
		}	
	}
	
	public synchronized void syncSource() throws InterruptedException {
	
		//for(int i=0; i<scans; i++) for(int k=0; k<subscan[i].length; k++) subscan[i][k].removeSource(reduced.signal, base);

		
		while(!queue.isEmpty()) wait();		
		
		syncThreads.clear();
		for(int i=0; i<useCPUs; i++) {
			SourceSyncThread thread = new SourceSyncThread(this);
			syncThreads.add(thread);
		}
			
		for(int i=0; i<scans; i++) syncThreads.get(i % useCPUs).addScan(subscan[i]);
		
		for(int i=1; i<useCPUs && i<syncThreads.size();) {
			if(syncThreads.get(i).scans.size() == 0) syncThreads.remove(i);
			else i++;
		}
		
		for(SourceSyncThread thread : syncThreads) thread.start();
		
		while(!queue.isEmpty()) wait();

		
	}
	
	
	
	
	

	
	
	
	
	
	
	
	
	
	
	public void summarize(LabocaScan subscan) {		
		System.err.print("   [" + (subscan.serialNo) + "|" + (subscan.subscanNo+1) + "] ");
		System.err.println(subscan.comments);
	}



	


	
	
	
	
	
	
	
	
	public Vector<SourceBoxingThread> boxingThreads = new Vector<SourceBoxingThread>();
	
	public synchronized void searchCorners() throws InterruptedException {
		while(!queue.isEmpty()) wait();		

		boxingThreads.clear();
		for(int i=0; i<useCPUs; i++) {
			SourceBoxingThread thread = new SourceBoxingThread(this);
			boxingThreads.add(thread);
		}

		for(int i=0; i<scans; i++) boxingThreads.get(i % useCPUs).addScan(subscan[i]);

		for(int i=1; i<useCPUs && i<boxingThreads.size();) {
			if(boxingThreads.get(i).scans.size() == 0) boxingThreads.remove(i);
			else i++;
		}

		for(SourceBoxingThread thread : boxingThreads) thread.start();

		while(!queue.isEmpty()) wait();			
	}
	
	public void createReduced() {
		System.out.print(" Initializing Source Map. ");

		if(options.containsKey("grid")) {
			resolution = Double.parseDouble(options.get("grid")) * Unit.arcsec;
		}

		map = new SkyMap();
		map.isHorizontal = options.containsKey("altaz");

		if(options.containsKey("jansky")) map.janskyPerMicroV = Double.parseDouble(options.get("jansky"));
		else map.janskyPerMicroV = 5.33;

		for(int i=0; i<scans; i++) map.scans.add(subscan[i][0]);	
		map.delta = resolution;
		map.beamFWHM = beamFWHM;
		map.creator = getClass().getSimpleName();
		map.creatorVersion = version;
		map.sourceName = subscan[0][0].sourceName;
		map.horizontal = subscan[0][0].horizontal;
		map.equatorial = subscan[0][0].equatorial;
		
		try { map.projection = options.containsKey("projection") ? SphericalProjection.forName(options.get("projection")) : new Gnomonic(); }
		catch(Exception e) { map.projection = new Gnomonic(); }
		
		map.projection.setReference(map.isHorizontal ? map.horizontal : map.equatorial);		
		map.init();
		
		// Figure out what offsets the corners of the map will have...
		try { searchCorners(); }
		catch(InterruptedException e) { 
			e.printStackTrace(); 
			System.exit(1);
		}

		// Make the reference fall on pixel boundaries.
		map.i0 = 0.5 - Math.round(SourceBoxingThread.minOffset.x/resolution);
		map.j0 = 0.5 - Math.round(SourceBoxingThread.minOffset.y/resolution);

		int sizeX = (int)Math.ceil((SourceBoxingThread.maxOffset.x - SourceBoxingThread.minOffset.x)/resolution) + 2;
		int sizeY = (int)Math.ceil((SourceBoxingThread.maxOffset.y - SourceBoxingThread.minOffset.y)/resolution) + 2;
		
		map.setSize(sizeX, sizeY);
		map.shortInfo();	
	}

	public void init() {
		scanSetup();
		if(!(options.containsKey("chopped") || options.containsKey("photometry"))) createReduced();
		
		if(options.containsKey("sources")) {
			String fileName = options.get("sources");
			for(int i=0; i<scans; i++) for(int k=0; k<subscan[i].length; k++) {
				System.out.print("[" + subscan[i][k].serialNo + "|" + subscan[i][k].subscanNo +"] ");
				try { DataSimulator.addSources(subscan[i][k], fileName); }
				catch(IOException e) { System.err.println("WARNING! Invalid maskfile. " + fileName); }
			}
		}
	}
		
	public void logScans() throws IOException {
		PrintStream out = new PrintStream(new FileOutputStream(workPath + File.separator + "scans.log", true));
		
		for(int i=0; i<scans; i++) for(int k=0; k<subscan[i].length; k++) {
			if(subscan[i][k].goodChannelCount == 0 || Double.isNaN(subscan[i][k].weight)) out.print("#");
			
			out.print(subscan[i][k].serialNo + " " + (subscan[i][k].subscanNo+1));
			out.print("\t" + Util.f4.format(subscan[i][k].getMJD()));
			out.print("\t" + subscan[i][k].goodChannelCount);
			out.print("\t" + Util.e3.format(subscan[i][k].nefd));
			out.print("\t" + Util.e3.format(subscan[i][k].sourceNEFD));
			out.print("\t" + Util.e3.format(subscan[i][k].weight));
			out.print("\t" + Util.f3.format(subscan[i][k].frame[0].transmission));
			out.print("\t" + Util.e3.format(subscan[i][k].rmsHe3));
			
			out.println();
		}
		
		out.println("#");
		
		out.flush();
		out.close();
		
	}
	
	public void addNoiseStats() {
		try {
			PrintStream out = new PrintStream(new FileOutputStream(workPath + File.separator + "noisestat.dat", true));

			out.println("# " + map.sourceName);

			for(int i=0; i<scans; i++) for(int k=0; k<subscan[i].length; k++) {

				double varCorr = 0.0, varSky = 0.0;
				int n = 0;

				float[] correlated = subscan[i][k].arrayCorrelated;
				float[] skyX = subscan[i][k].skyX;
				float[] skyY = subscan[i][k].skyY;
				
				
				for(int t=0; t<subscan[i][k].frames; t++) {
					LabocaFrame exposure = subscan[i][k].frame[t];
					if(exposure != null) if((exposure.flag & LabocaFrame.SOURCE_FLAGS) == 0) {
						varCorr += correlated[t] * correlated[t];
						varSky += skyX[t] * skyX[t] + skyY[t] * skyY[t];
						n++;
					}
				}
					
				double skynoise = Math.sqrt(varCorr/n) / (n*subscan[i][k].integrationTime / Unit.min);
				double gradientNoise = Math.sqrt(varSky/n) / (threads.get(0).getFilterT(subscan[i][k], "drifts", 10.0*Unit.sec) / Unit.min) * Unit.arcmin;

				double transmission = 0.5 * (subscan[i][k].frame[0].transmission + subscan[i][k].frame[subscan[i][k].frames - 1].transmission);


				out.print(subscan[i][k].serialNo + "\t" + (k+1));
				out.print("\t" + Util.f3.format(1.0 - transmission));
				out.print("\t" + Util.e3.format(subscan[i][k].nefd));
				out.print("\t" + Util.e3.format(skynoise));
				out.print("\t" + Util.e3.format(gradientNoise));
				out.println();
			}

			out.flush();
			out.close();
		}
		catch(IOException e) {}
	}
	
	public void printConfig(PrintStream out) {
		Vector<String> keys = new Vector<String>();
		Hashtable<String, String> configuration = getConfiguration();
		for(String key : configuration.keySet()) keys.add(key);
		Collections.sort(keys);
		out.println("# " + getClass().getSimpleName() + " configuration settings");
		out.println("# -----------------------------------------------------------------");
		for(String key : keys) out.println("  " + key + " " + configuration.get(key));		
		out.println("# -----------------------------------------------------------------");
	}

	public static void info() {
		String info = "\n" +
			"  ----------------------------------------------------------------------\n" +
			"  minicrush " + version + " by Attila Kovacs <akovacs@mpifr-bonn.mpg.de>\n" +
			"  APEX bolometer array data reduction tool.\n" +
			"  ----------------------------------------------------------------------\n";	
		System.err.println(info);
	}

	public static void usage() {
		String info = "  Usage: minicrush instrument [options] scanslist [[options] scanlist ...]\n" +
			"\n" +
			"    instrument      'laboca' or 'aszca'\n" +
			"    [options]       various configuration options. See README for details.\n" +
			"    scanlist        A list of scan numbers to reduce. Can be composed from\n" +
			"                    individual scans or ranges. E.g.\n" +
			"                      10628-10633 11043 12641-12647\n" +
			"\n" +
			"   Try 'minicrush [laboca|aszca] -poll' for a list of current settings.\n" +
			"   or 'minicrush -help' for a brief list of commonly used options.\n"; 
		
		System.out.println(info);
	}
	
	public static void help() {
		String info = "\n" +
			" Some commonly used options. For full and detailed description of all options.\n" +
			" please consult README.\n\n" +
			"   Location of raw data:\n" +
			"\t-project=    Specify the project ID (upper case).\n" +
			"\t-datapath=   Specify the path to the raw data.\n" +
			"   Optimize reduction for source type:\n" +
			"\t-bright      Reduce bright sources (S/N > 1000).\n" +
			"\t-faint       Use with faint sources (S/N < 10).\n" +
			"\t-deep        Use with deep fields (point sources).\n" +
			"\t-extended    Assume extended structures.\n" +
			"\t-sourcesize= Specify a sourcesize in arcsec (with -faint or -deep).\n" + 
			"\t-outpath=    Specify the directory where output files will go.\n" +
			"   Options for the output map:\n" +
			"\t-altaz       Reduce in horizontal coordinates (for pointing).\n" +
			"\t-name=       Specify the output FITS map file name (rel. to outpath).\n" +
			"\t-projection= The spherical projection to use (e.g. SIN, TAN, SFL...)\n" +
			"\t-grid=       The map pixelization (arcsec).\n" +
			"\t-smooth=     Smooth map by the specified beam (arcsec).\n" +
			"   Commonly used options for scans:\n" +
			"\t-tau=        Specify a zenith tau to use.\n" +
			"\t-scale=      Apply a calibration factor to the scan(s).\n" +
			"\t-center=     Apply pointing corrections to scan.\n" +
			"   Other useful options:\n" +
			"\t-config=     Load configiration file.\n" +
			"\t-poll        Poll the currently set options.\n" +
			"\t-forget=     Comma separated list of options to unset.\n" +
			"\t-blacklist=  Comma separated list of options to completely disallow.\n" +
			"\n";
		
		System.out.println(info);
	}
	
	public static String checkReleaseVersion() {       
		try {
			URL versionURL = new URL("http://www.submm.caltech.edu/~sharc/crush/mini/release.version");
			URLConnection connection = versionURL.openConnection();
			try {
				connection.setConnectTimeout(3000);
				connection.setReadTimeout(3000);
				BufferedReader in = new BufferedReader(new InputStreamReader(connection.getInputStream()));
				return in.readLine();
			} 
			catch(IOException e) {
				System.err.println("WARNING! Timed out while awaiting version update information.");
			}
		}
		catch(MalformedURLException e) { e.printStackTrace(); }
		catch(IOException e) {
			System.err.println("WARNING! No connection to version update server.");
		}

		return null;
	}
	
	public static int getMajorNumber(String version) {
		StringTokenizer tokens = new StringTokenizer(version, ".");
		return Integer.parseInt(tokens.nextToken());
	}

	public static int getMinorNumber(String version) {
		StringTokenizer tokens = new StringTokenizer(version, ".-");
		tokens.nextToken();
		return Integer.parseInt(tokens.nextToken());
	}

	public static String getCompilationString(String version) {
		StringTokenizer tokens = new StringTokenizer(version, "-");
		tokens.nextToken();
		return tokens.hasMoreTokens() ? tokens.nextToken() : "0";
	}   
	
	public static void checkForUpdates() {	
		String releaseVersion = checkReleaseVersion();
		if(releaseVersion == null) return;

		if(!version.equals(releaseVersion)) {
			boolean isNew = false;

			if(getMajorNumber(version) < getMajorNumber(releaseVersion)) isNew = true; 
			else if(getMinorNumber(version) < getMinorNumber(releaseVersion)) isNew = true;
			else {
				String compString = getCompilationString(version);
				String releaseCompString = getCompilationString(releaseVersion);
				int releasecomp = -1;
				int comp = -1;
				
				try { comp = Integer.parseInt(compString); } catch(NumberFormatException e) {}
				try { releasecomp = Integer.parseInt(releaseCompString); } catch(NumberFormatException e) {}
				
				if(releasecomp > comp) isNew = true;
				else if(releasecomp == comp && compString.compareTo(releaseCompString) < 0) isNew = true;
			}

			if(isNew) {
				System.err.println();


				for(int i=0; i<8; i++) System.err.print("**********");
				System.err.println();

				System.err.println("[UPDATE] Version Update Information! ");		
				System.err.println("[UPDATE] A NEW MINICRUSH RELEASE IS NOW AVAILABLE FOR DOWNLOAD!!! "); 
				System.err.println("[UPDATE] from www.submm.caltech.edu/~sharc/crush/download.htm");
				System.err.println("[UPDATE] Version: " + releaseVersion);

				for(int i=0; i<8; i++) System.err.print("**********");
				System.err.println(); 		

				System.err.println();
			}
		}
	}

	public static void getMessages() {
		try {
			URL messageURL = new URL("http://www.submm.caltech.edu/~sharc/crush/mini/messages");
			URLConnection connection = messageURL.openConnection();
			try {
				connection.setConnectTimeout(3000);
				connection.setReadTimeout(3000);
				BufferedReader in = new BufferedReader(new InputStreamReader(connection.getInputStream()));
				String line = "";

				System.err.println();
				System.err.println("[www] Message from www.submm.caltech.edu/~sharc/crush/mini: ");
				while((line = in.readLine()) != null) System.err.println("[CRUSH] " + line);
				System.err.println();
			}
			catch(IOException e) {
				System.err.println("WARNING! Timed out while awaiting messages from CRUSH server.");
			}
		} 
		catch(MalformedURLException e) { e.printStackTrace(); }
		catch(IOException e) {
			System.err.println("WARNING! No connection to messages server.");
		}
	}
}

