/*******************************************************************************
 * Copyright (c) 2010 Attila Kovacs <attila_kovacs@post.harvard.edu>.
 * All rights reserved. 
 * 
 * This file is part of crush.
 * 
 *     crush is free software: you can redistribute it and/or modify
 *     it under the terms of the GNU General Public License as published by
 *     the Free Software Foundation, either version 3 of the License, or
 *     (at your option) any later version.
 * 
 *     crush is distributed in the hope that it will be useful,
 *     but WITHOUT ANY WARRANTY; without even the implied warranty of
 *     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 *     GNU General Public License for more details.
 * 
 *     You should have received a copy of the GNU General Public License
 *     along with crush.  If not, see <http://www.gnu.org/licenses/>.
 * 
 * Contributors:
 *     Attila Kovacs <attila_kovacs@post.harvard.edu> - initial API and implementation
 ******************************************************************************/

// Copyright (c) 2007,2008,2009,2010 Attila Kovacs

package crush;

import java.io.*;
import java.net.*;
import java.util.*;

import util.*;

/**
 * 
 * @author Attila Kovacs
 * @version 2.00-b3
 * 
 */
public class CRUSH extends Configurator {
	public static String version = "2.00-b3";
	public static String workPath = ".";
	public static String home = ".";
	
	public Instrument<?> instrument;
	public SourceModel<?,?> source;
	public Vector<Scan<?,?>> scans = new Vector<Scan<?,?>>();
	public String commandLine;
	
	public static int useCPUs = 1;
	
	public static void main(String[] args) {
		info();
		
		Locale.setDefault(Locale.US);
		
		if(args.length == 0) {
			usage();
			System.exit(0);			
		}
		
		if(args[0].equalsIgnoreCase("-help")) {
			help();
			System.exit(0);	
		}
		
		home = System.getenv("CRUSH");
		if(home == null) home = ".";
		
		CRUSH crush = new CRUSH(args[0]);
		crush.init(args);
		
		try { crush.reduce(); }
		catch(InterruptedException e) {}
	}

	public CRUSH(String instrumentName) {
		checkForUpdates();
		
		instrument = Instrument.forName(instrumentName.toLowerCase());
		
		if(instrument == null) {
			System.err.println("Warning! Unknown instrument " + instrumentName);
			System.exit(1);
		}
		
		System.err.println("Instrument is " + instrument.name.toUpperCase());
		instrument.options = this;
	}
	
	public void init(String[] args) {
		readConfig("default.cfg");
		commandLine = args[0];
		
		for(int i=1; i<args.length; i++) {
			commandLine += " " + args[i]; 
			
			if(args[i].charAt(0) == '-') {
				String option = args[i].substring(1);
				parse(option);
				
				if(option.equals("bright")) readConfig("bright.cfg"); 
				else if(option.equals("faint")) readConfig("faint.cfg"); 
				else if(option.equals("deep")) readConfig("deep.cfg"); 
				else if(option.equals("extended")) readConfig("extended.cfg");
				else if(option.equals("photometry")) readConfig("photometry.cfg");
			}
			else {
				if(!instrument.initialized) instrument.initialize();
				read(args[i]);
			}
		}	

		validate();
	}
	

	@Override
	public void readConfig(String fileName) {
		String userConfPath = System.getProperty("user.home") + File.separator + ".crush2"+ File.separator;
		
		try { super.readConfig(CRUSH.home + File.separator + fileName); }
		catch(IOException e) {}
		
		try { super.readConfig(userConfPath + fileName); }
		catch(IOException e) {}
		
		try { super.readConfig(instrument.getDefaultConfigPath() + fileName); }
		catch(IOException e) { }
		
		// read the instrument overriderrs (if any).
		try { super.readConfig(userConfPath + instrument.name + File.separator + fileName); }
		catch(IOException e) {}
	}
	
	public void validate() {
		if(scans.size() == 0) {
			System.err.println("No scans to reduce. Exiting.");
			System.exit(1);
		}
		
		instrument.initialize();
		
		// Keep only the non-specific global options here...
		for(Scan<?,?> scan : scans) instrument.options.intersect(scan.instrument.options); 
		
		if(containsKey("outpath")) workPath = Util.getSystemString(get("outpath").getValue()); 
		
		for(int i=0; i<scans.size(); i++) if(scans.get(i).isEmpty()) scans.remove(i--);
		
		Collections.sort(scans);
		
		refreshUseCPUs();
		System.err.println("Will use " + Math.min(scans.size(), useCPUs) + " CPU core(s).");
	}
	
	public void refreshUseCPUs() {
		useCPUs = Runtime.getRuntime().availableProcessors();
		if(isConfigured("reservecpus")) useCPUs -= get("reservecpus").getInt();
		useCPUs = Math.max(useCPUs, 1);
	}

	public void read(String scanID) {
		StringTokenizer list = new StringTokenizer(scanID, "; \t");
		refreshUseCPUs();
		
		if(list.countTokens() > 1) {
			while(list.hasMoreTokens()) read(list.nextToken());
		}
		else { 
			try {
				Scan<?,?> scan = (Scan<?,?>) instrument.getScanInstance();
				if(isConfigured("obslog")) {
					scan.readHeader(scanID);
					scan.writeObsLog();
				}
				else { 
					scan.read(scanID);
					scans.add(scan);	
					System.gc();		
				}
				System.err.println();
			}
			catch(OutOfMemoryError e) {
				System.err.println("ERROR! Ran of of memory while reading scan.");
				System.err.println();
				System.err.println("   * Increase the amount of memory available to crush, by editing the '-Xmx'");
				System.err.println("     option to Java in 'wrapper.sh' (or 'wrapper.bat' for Windows).");
				System.err.println();
				System.err.println("   * If using 64-bit Unix OS and Java, you can also add the '-d64' option to");
				System.err.println("     allow Java to access over 2GB.");
				System.err.println();
				System.err.println("   * Try reduce scans in smaller chunks. You can then use 'coadd' to combine");
				System.err.println("     the maps post reduction. Note: it is always preferable to try reduce all");
				System.err.println("     scans together, if there is a way to fit them into memory.");
			    System.err.println();
			    System.exit(1);				
			}
			catch(FileNotFoundException e) { 
				// If it has '-', try to see if it can be read as a range...
				if(scanID.contains("-")) {
					try { 
						StringTokenizer range = new StringTokenizer(scanID, "-");
						int from = Integer.parseInt(range.nextToken());
						int to = Integer.parseInt(range.nextToken());
						for(int no = from; no <= to; no++) read(no + "");
					}
					catch(Exception parseError) {
						System.err.println(" ERROR! " + parseError.getMessage()); 
					}
				}
				else System.err.println("\n ERROR! " + e.getMessage() + "\n");
			}
			catch(IOException e) { System.err.println(" ERROR! " + e.getMessage()); }
			catch(Exception e) { e.printStackTrace(); }
		}	
	}


	public void reduce() throws InterruptedException {	
		int rounds = 0;
	
		if(isConfigured("bright")) System.out.println("Bright source reduction.");
		else if(isConfigured("faint")) System.out.println("Faint source reduction.");
		else if(isConfigured("deep")) System.out.println("Deep source reduction.");
		else System.out.println("Default reduction.");
		
		if(isConfigured("extended")) System.out.println("Assuming extended source(s).");
	
		System.out.println("Assuming " + Util.f1.format(instrument.getSourceSize()/instrument.getDefaultSizeUnit()) + " " + instrument.getDefaultSizeName() + " sized source(s).");
		
		System.out.println();
		
		source = ((Instrument<?>) instrument.copy()).getSourceModelInstance();
		
		if(source != null) {
			source.commandLine = commandLine;
			source.options = this;
			source.create(scans);
		}
		
		if(isConfigured("rounds")) rounds = get("rounds").getInt();
		
		for(int iteration=1; iteration<=rounds; iteration++) {
			System.err.println();
			System.err.println(" Round " + iteration + ": ");	

			setIteration(iteration, rounds);	
			
			for(Scan<?,?> scan : scans) {
				scan.instrument.options.setIteration(iteration, rounds);	
				for(Integration<?,?> integration : scan) if(integration.instrument != scan.instrument)  
				integration.instrument.options.setIteration(iteration, rounds);	
			}
			
			iterate();	
		}

		System.err.println();
		
		
		if(source != null) {
			try { source.write(workPath); }
			catch(Exception e) { e.printStackTrace(); }
 		}
		
		for(Scan<?,?> scan : scans) for(Integration<?,?> integration : scan) integration.writeProducts();

	}
	
	
	public synchronized void iterate() throws InterruptedException {
		Vector<String> ordering = get("ordering").getLowerCaseList();
		Vector<String> tasks = new Vector<String>();
		
		for(String task : ordering) {
			tasks.add(task);
			if(task.startsWith("source")) if(isConfigured("source")) if(source != null) {
				iterate(tasks);
				tasks.clear();
			}
		}
		
		if(!tasks.isEmpty()) iterate(tasks);
		
		System.err.println();
	}
	
	public synchronized void iterate(Vector<String> tasks) throws InterruptedException {
		while(!queue.isEmpty()) wait();
		
		Vector<Pipeline> threads = new Vector<Pipeline>();
		System.err.println();
			
		threads.clear();	
		
		for(int i=0; i<useCPUs; i++) {
			Pipeline thread = new Pipeline(this);
			thread.setOrdering(tasks);
			threads.add(thread);
		}
		
		for(int i=0; i<scans.size(); i++) threads.get(i % useCPUs).addScan(scans.get(i));
		
		for(int i=1; i<useCPUs && i<threads.size();) {
			if(threads.get(i).scans.size() == 0) threads.remove(i);
			else i++;
		}

		
		if(tasks.contains("source")) if(isConfigured("source")) source.reset();
				
		for(Pipeline thread : threads) thread.start();
	
		summarize();
		
		if(tasks.contains("source")) if(isConfigured("source")) {
			System.err.print("  [Source] ");
			source.sync();
		}
		
			
		if(isConfigured("whiten")) if(get("whiten").isConfigured("once")) remove("whiten");
		
		System.gc();
	}

	Vector<Integration<?, ?>> queue = new Vector<Integration<?, ?>>();
	
	public synchronized void checkout(Integration<?,?> integration) {
		queue.remove(integration);
		notifyAll();
	}
	
	public synchronized void summarize() throws InterruptedException {
		// Go in order.
		// If next one dissappears from queue then print summary
		// else wait();
		
		for(Scan<?, ?> scan : scans) for(Integration<?,?> integration : scan) {
			while(queue.contains(integration)) wait();
			summarize(integration);
			notifyAll();
		}	
	}

	
	public void summarize(Integration<?,?> integration) {		
		System.err.print("  [" + (integration.scan.serialNo) + "|" + (integration.integrationNo+1) + "] ");
		System.err.println(integration.comments);
		integration.comments = new String();
	}	
	

	public static void info() {
		String info = "\n" +
			"  ----------------------------------------------------------------------\n" +
			"  crush -- Reduction and imaging tool for bolometer arrays.\n" +
			"           Version " + version + "\n" +
			"           http://www.submm.caltech.edu/~sharc/crush\n" +
			"           Copyright (C)2010 Attila Kovacs <kovacs[AT]astro.umn.edu>\n" +
			"  ----------------------------------------------------------------------\n";	
		System.err.println(info);
	}

	public static void usage() {
		String info = "  Usage: crush <instrument> [options] <scanslist> [[options] <scanlist> ...]\n" +
			"\n" +
			"    <instrument>    'aszca', 'ebex', 'gismo', 'laboca', 'p-artemis', 'polka',\n" +
			"                    'saboca' or 'sharc2'.\n" +
			"    [options]       various configuration options. See README for details.\n" +
			"    <scanlist>      A list of scan numbers (or names) to reduce. Can mix\n" +
			"                    file names, individual scan numbers, and ranges. E.g.\n" +
			"                       10628-10633 11043 myscan.fits\n" +
			"\n" +
			"   Try 'crush <instrument> -poll' for a list of current settings.\n" +
			"   or 'crush -help' for a brief list of commonly used options.\n"; 
		
		System.out.println(info);
	}
	
	// TODO Update most common options
	public static void help() {
		String info = "\n" +
			" Some commonly used options. For full and detailed description of all options.\n" +
			" please consult the GLOSSARY.\n\n" +
			"   Location of raw data:\n" +
			"\t-project=    Specify the project ID (APEX only) in upper case.\n" +
			"\t-datapath=   Specify the path to the raw data.\n" +
			"   Optimize reduction by source type:\n" +
			"\t-bright      Reduce bright sources (S/N > 1000).\n" +
			"\t-faint       Use with faint sources (S/N < 10).\n" +
			"\t-deep        Use with deep fields (point sources).\n" +
			"\t-extended    Assume extended structures.\n" +
			"\t-outpath=    Specify the directory where output files will go.\n" +
			"   Options for the output map:\n" +
			"\t-altaz       Reduce in horizontal coordinates (for pointing).\n" +
			"\t-name=       Specify the output FITS map file name (rel. to outpath).\n" +
			"\t-projection= The spherical projection to use (e.g. SIN, TAN, SFL...)\n" +
			"\t-grid=       The map pixelization (arcsec).\n" +
			"\t-smooth=     Smooth map by the specified beam (arcsec).\n" +
			"   Commonly used options for scans:\n" +
			"\t-tau=        Specify a zenith tau to use.\n" +
			"\t-scale=      Apply a calibration factor to the scan(s).\n" +
			"\t-center=     Apply pointing corrections to scan.\n" +
			"   Other useful options:\n" +
			"\t-config=     Load configuration file.\n" +
			"\t-poll        Poll the currently set options.\n" +
			"\t-forget=     Comma separated list of options to unset.\n" +
			"\t-blacklist=  Comma separated list of options to ignore.\n" +
			"\n";
		
		System.out.println(info);
	}
	
	public static String getReleaseVersion() {       
		try {
			URL versionURL = new URL("http://www.submm.caltech.edu/~sharc/crush/v2/release.version");
			URLConnection connection = versionURL.openConnection();
			try {
				connection.setConnectTimeout(3000);
				connection.setReadTimeout(3000);
				BufferedReader in = new BufferedReader(new InputStreamReader(connection.getInputStream()));
				return in.readLine();
			} 
			catch(IOException e) {
				System.err.println("WARNING! Timed out while awaiting version update information.");
			}
		}
		catch(MalformedURLException e) { e.printStackTrace(); }
		catch(IOException e) {
			System.err.println("WARNING! No connection to version update server.");
		}

		return null;
	}
	  
	
	public static void checkForUpdates() {	
		String releaseVersion = getReleaseVersion();
		if(releaseVersion == null) return;
		
		VersionInformation release = new VersionInformation(releaseVersion);
		if(release.compareTo(new VersionInformation(version)) <= 0) return; 

		for(int i=0; i<8; i++) System.err.print("**********");
		System.err.println();
		
		System.err.println("  A NEW CRUSH-2 RELEASE IS NOW AVAILABLE FOR DOWNLOAD!!! "); 
		System.err.println();
		System.err.println("  Version: " + releaseVersion);
		System.err.println();
		System.err.println("  Get it from:  www.submm.caltech.edu/~sharc/crush");
		System.err.println();
		System.err.println("  You should always update to the latest release to take advantage of critical");
		System.err.println("  bug fixes, improvements and new features.");
		
		for(int i=0; i<8; i++) System.err.print("**********");
		System.err.println();
		
		System.err.println();
		
		for(int i=5; i>0; i--) {
			System.err.print("\rWill continue in " + i + " seconds.");
			try { Thread.sleep(1000); }
			catch(InterruptedException e) {};
		}
		System.err.println("\rContinuing...                          ");
		
		System.err.println();
	}

	/*
	public static void printMessages() {
		try {
			URL messageURL = new URL("http://www.submm.caltech.edu/~sharc/crush/v2/messages");
			URLConnection connection = messageURL.openConnection();
			try {
				connection.setConnectTimeout(3000);
				connection.setReadTimeout(3000);
				BufferedReader in = new BufferedReader(new InputStreamReader(connection.getInputStream()));
				String line = "";

				System.err.println();
				System.err.println("[www] Message from www.submm.caltech.edu/~sharc/crush/v2: ");
				while((line = in.readLine()) != null) System.err.println("[CRUSH] " + line);
				System.err.println();
			}
			catch(IOException e) {
				System.err.println("WARNING! Timed out while awaiting messages from CRUSH server.");
			}
		} 
		catch(MalformedURLException e) { e.printStackTrace(); }
		catch(IOException e) {
			System.err.println("WARNING! No connection to messages server.");
		}
	}
	*/
}

