// (C)2007 Attila Kovacs <attila@submm.caltech.edu>

import java.io.*;
import java.util.*;


import nom.tam.fits.*;
import nom.tam.util.*;


public class LabocaScan implements Cloneable {
	Hashtable<String, String> options;
	LabocaFrame[] frame;
	LabocaPixel[] channel;
	int channels, frames;	

	Hashtable<String, ModelParms> modelParms = new Hashtable<String, ModelParms>(); 
	
	LabocaPixel channelTemplate = new LabocaPixel(-1);
	
	EquatorialCoordinates equatorial;
	HorizontalCoordinates horizontal;
	
	boolean isNativeEquatorial = false;
	
	double PA;
	String sourceName;
	String scanPath;
	String timeStamp;
	String descriptor;
	String comments = new String();
	int serialNo, subscanNo;
	
	double beamFWHM = 9.0*Unit.arcsec;
	
	boolean debug = false;
	double instrumentGain;
	double janskyPerMicroV;

	
	SkyMap map;

	boolean approximateSourceMap = false, extendedFilter = false;
	int sourceGeneration = 0;
	double extendedScale = 120.0 * Unit.arcsec;

	double minScanSpeed = Double.NaN;
	double maxScanSpeed = Double.NaN;
	double maxAcceleration = Double.NaN;
	double aveScanSpeed = Double.NaN;
	
	int referenceChannel;

	double integrationTime = 1.0;

	int onPhase = 1;
	int offPhase = 2;

	boolean isChopped = false;
	double chopperThrow = 0.0;

	double gain = 1.0;
	double zenithTau = 1.0;
	double nefd = Double.NaN; // It is readily cast into the Jy sqrt(s) units!!!
	double sourceNEFD = Double.NaN;
	double weight = 1.0;
	
	int[] allChannels, allSkyChannels;
	int[][] boxChannels, cableChannels, ampChannels, waferChannels, regionChannels;
	
	boolean useHe3Data, calcHe3Gains;		
	double he3TimeScale = 3.0 * Unit.sec;
	
	boolean useBlindData, calcBlindGains;		
	double blindTimeScale = 3.0 * Unit.sec;
	
	double rmsHe3 = 0.0;
	
	boolean uniformGains = false;
	
	double hipassTimeScale = Double.NaN;
	boolean scramble = false;
	int driftN = 128; // The 1/f filtering timescale (in frames), or its default value to be assumed...
	
	int goodChannelCount = -1;
	
	public String instrumentName = "laboca";
	public String rcpFile;
	public String pixelDataFile = instrumentName + File.separator + "pixel.dat";
	
	public double pixelSeparation = 36.0 * Unit.arcsec;
	
	public Vector<Integer> flaggedPixels = new Vector<Integer>();
	
	public LabocaScan() {	}
	
	public Object clone() { 
		try { 
			LabocaScan clone = (LabocaScan) super.clone(); 
			clone.reuseOffset = new Vector2D();
			clone.offsetData = null;
			clone.corrData = null;
			clone.gainData = null;
			clone.modelParms = new Hashtable<String, ModelParms>(); 
			return clone;
		}	
		catch(CloneNotSupportedException e) { return null; }	
	}
	
	public LabocaScan copy() {
		LabocaScan copy = (LabocaScan) clone();
		copy.channel = new LabocaPixel[channel.length];
		for(int c=0; c<channel.length; c++) copy.channel[c] = (LabocaPixel) channel[c].clone();
		copy.comments = new String(comments);
		return copy;
	}
	
	public void validate() {	
		// Finish with updating info on the channels...
		
		if(calcHe3Gains) {
			try { getHe3TemperatureGains(); } 
			catch(IOException e) {}	
		}
		
		if(useHe3Data) {
			try { correctForTemperatureDrifts(); } 
			catch(IOException e) {
				System.err.println("   WARNING! " + "He3 gains not read. Skipping correction.");
			}	
		}
		
		else if(useBlindData) {
			try { correctViaBlinds(); } 
			catch(IOException e) {
				System.err.println("   WARNING! " + "He3 gains not read. Skipping correction.");				
			}	
		}
		
		if((!Double.isNaN(minScanSpeed) && minScanSpeed > 0.0) || (!Double.isNaN(maxScanSpeed) && maxScanSpeed > 0.0)) {
			System.err.print("   Clipping frames with unsuitable scanning speeds. ");
			System.err.println(" [" + (int)Math.round(100.0*(float)velocityCut(minScanSpeed, maxScanSpeed)/frames) + "% clipped.]");
		}
		if(!Double.isNaN(maxAcceleration) && maxAcceleration > 0.0) {
			System.err.print("   Clipping frames with high telescope acceleration. ");
			System.err.println(" [" + (int)Math.round(100.0*(float)accelerationCut(maxAcceleration)/frames) + "% clipped.]");
		}	
		aveScanSpeed = getAverageScanningVelocity(0.5 * Unit.s);
		System.err.print("   Median Scanning speed is");
		System.err.println(" " + Util.f1.format(aveScanSpeed/(Unit.arcmin/Unit.s)) + " arcmin/s");
		
		try { 
			if(rcpFile == null) rcpFile = getCorrespondingRCP(instrumentName + File.separator + "rcps.dat");	
			LabocaPixel.readPixelRCP(channel, rcpFile, channel[referenceChannel]);
			System.err.println("   Updated pixel position data from " + rcpFile);
		} catch(IOException e) { System.err.println("ERROR! " + e.getMessage()); }
		

		try {
			LabocaPixel.loadPixelData(channel, pixelDataFile);
			System.err.println("   Loaded pixel defaults from " + pixelDataFile);
		} catch(IOException e) { System.err.println(" ERROR! " + e.getMessage()); }
		
		
		if(!flaggedPixels.isEmpty()) {
			System.err.println("   Flagging " + flaggedPixels.size() + " bad channels.");
			for(int c=0; c<channels; c++) if(flaggedPixels.contains(channel[c].backendIndex + 1)) channel[c].flag |= LabocaPixel.FLAG_DEAD;
		}
		
		try { flagCorrespondingBadChannelList(instrumentName + File.separator + "flagging.dat"); }
		catch(IOException e) {}
		
		try { LabocaPixel.readWiring(instrumentName + File.separator + "wiring.dat"); }
		catch(IOException e) {
			System.err.println("ERROR! Problem reading wiring data. Exiting.");
			System.exit(1);
		}
		
		LabocaPixel.fillWiringInfo(channel);
		System.err.println("   Loaded pixel wiring data.");
		
		if(uniformGains) {
			for(int c=0; c<channels; c++) {
				channel[c].gain = 1.0;
				channel[c].mainBeamEfficiency = 1.0;
			}
		}
		
		LabocaPixel.kill(channel, LabocaPixel.FLAG_BLIND);
		int[] blindChannels = getBlindChannels();
		for(int k=0; k < blindChannels.length; k++) {
			channel[blindChannels[k]].flag &= ~LabocaPixel.FLAG_DEAD;
			channel[blindChannels[k]].flag |= LabocaPixel.FLAG_BLIND;
		}
	
		trim();
		if(!options.containsKey("noslim")) slim();
		
		allChannels = LabocaPixel.getAllChannels(channel, LabocaPixel.FLAG_DEAD);
		allSkyChannels = LabocaPixel.getAllChannels(channel, LabocaPixel.FLAG_DEAD | LabocaPixel.FLAG_BLIND);
		
		try { getChannelGroups(); }
		catch(Exception e) { e.printStackTrace(); }
		
		for(int c=0; c<channels; c++) channel[c].dof = frames;
			
		LabocaPixel.findNeighbours(channel, 2.0/3.0*pixelSeparation, 1.5*pixelSeparation);
		System.err.println("   Registered pixel neighbours.");
		
		if(!Double.isNaN(hipassTimeScale)) {
			System.err.print("   Hi-pass Filtering...");
			highPassFilter(hipassTimeScale);
			System.err.println();
		}
		
		if(isChopped) markChopped();
	
	}

	public void getChannelGroups() throws Exception {
		boxChannels = LabocaPixel.getChannelGroups(channel, LabocaPixel.class.getField("box"), LabocaPixel.FLAG_DEAD); 
		cableChannels = LabocaPixel.getChannelGroups(channel, LabocaPixel.class.getField("cable"), LabocaPixel.FLAG_DEAD); 
		ampChannels = LabocaPixel.getChannelGroups(channel, LabocaPixel.class.getField("amp"), LabocaPixel.FLAG_DEAD); 

		pinOrder(cableChannels);
	}
	
	public double getMJD() {
		return 0.5*(frame[0].MJD + frame[frames-1].MJD);	
	}
	
	public String getCorrespondingRCP(String datafile) throws IOException { 
		BufferedReader in = new BufferedReader(new InputStreamReader(new FileInputStream(datafile)));
		
		double MJD = getMJD();
		
		String line;
		while((line = in.readLine()) != null) if(line.length() > 0) if(line.charAt(0) != '#') {
			StringTokenizer tokens = new StringTokenizer(line);
			double startMJD = Double.parseDouble(tokens.nextToken());
			double endMJD = Double.parseDouble(tokens.nextToken());
			if(MJD >= startMJD && MJD <= endMJD) return instrumentName + File.separator + tokens.nextToken();
		}
		
		return instrumentName + File.separator + "master.rcp";
	}
	
	
	public void flagCorrespondingBadChannelList(String datafile) throws IOException {
		Vector<Integer> list = new Vector<Integer>();
		BufferedReader in = new BufferedReader(new InputStreamReader(new FileInputStream(datafile)));
		
		double MJD = getMJD();
		
		String line;
		
		while((line = in.readLine()) != null) if(line.length() > 0) if(line.charAt(0) != '#') {
			StringTokenizer tokens = new StringTokenizer(line, " \t\n\r,");
			double startMJD = Double.parseDouble(tokens.nextToken());
			double endMJD = Double.parseDouble(tokens.nextToken());
			if(MJD >= startMJD && MJD <= endMJD) {
				while(tokens.hasMoreTokens()) {
					try { list.add(Integer.parseInt(tokens.nextToken())); }
					catch(NumberFormatException e) {}
				}
			}
		}
		
		if(list.isEmpty()) return;
		
		System.err.println("   Flagging " + list.size() + " cross-talkers.");
		
		for(int c=0; c<channels; c++) if(list.contains(channel[c].backendIndex + 1)) channel[c].flag |= LabocaPixel.FLAG_DEAD;
		
	}
	
	public void pinOrder(int[][] grouping) {
		for(int g=0; g<grouping.length; g++) {
			Vector<LabocaPixel> selection = new Vector<LabocaPixel>();
			for(int k=0; k<grouping[g].length; k++) selection.add(channel[grouping[g][k]]);
			
			Comparator<LabocaPixel> pinorder = new Comparator<LabocaPixel>() {
				public int compare(LabocaPixel c1, LabocaPixel c2) {
					if(c1.pin < c2.pin) return -1;
					else if(c1.pin > c2.pin) return 1;
					else return 0;
				}				
			};
			
			Collections.sort(selection, pinorder);
			
			for(int k=0; k<grouping[g].length; k++) grouping[g][k] = selection.get(k).index;		
		}
		System.err.println("   Groups rearranged to pin order.");	
	}
	
	public void setZenithTau(double value) {
		System.err.println("   Setting zenith tau to " + Util.f3.format(value));
		zenithTau = value;
		for(int t=0; t<frames; t++) if(frame[t] != null) frame[t].setZenithTau(zenithTau);
	}
	
	public synchronized void trim() {
		int first = 0, last = frames-1;
		while(frame[first] == null && first < frames-1) first++;
		while(frame[last] == null && last > first) last--;
	
		if(first == 0 && last == frames-1) return;
		
		LabocaFrame[] trimmed = new LabocaFrame[last-first+1];
		for(int from=first, to=0; from <= last; from++, to++) trimmed[to] = frame[from];
		
		frame = trimmed;
		frames = trimmed.length;	
		
		System.err.println("   Trimmed to " + frames + " frames.");
	}
	
	public synchronized void slim() {
		int good = 0;
		for(int c=0; c<channels; c++) if(channel[c].isUnflagged(LabocaPixel.FLAG_DEAD) || c == referenceChannel) good++;
		
		LabocaPixel[] goodChannel = new LabocaPixel[good];
		
		for(int c=0, k=0; c<channels; c++) if(channel[c].isUnflagged(LabocaPixel.FLAG_DEAD) || c == referenceChannel) goodChannel[k++] = channel[c];		
		
		//for(int k=0; k<goodChannel.length; k++) System.err.println(goodChannel[k].index + " ---> " + k);
		
		for(int t=0; t<frames; t++) if(frame[t] != null) frame[t].slim(goodChannel);
		
		channel = goodChannel;		
		channels = channel.length;	
		
		for(int c=0; c<channels; c++) {
			if(channel[c].backendIndex == referenceChannel) referenceChannel = c;
			channel[c].index = c;
		}
	
		
		System.err.println("   Slimmed to " + channels + " working pixels.");
	}
	
	public synchronized void scale(double factor) {
		for(int t=0; t<frames; t++) if(frame[t] != null) for(int c=0; c<channels; c++) frame[t].data[c] *= factor;
	}
	
	public double getMax() {
		double max = Double.NEGATIVE_INFINITY;
		for(int t=0; t<frames; t++) if(frame[t] != null) for(int c=0; c<channels; c++) if(frame[t].data[c] > max) max = frame[t].data[c];
		return max;		
	}
	
	public int[][] getChannelGroup(String name) {
		if(name.equalsIgnoreCase("all")) return new int[][] { allChannels };
		else if(name.equalsIgnoreCase("boxes")) return boxChannels;
		else if(name.equalsIgnoreCase("cables")) return cableChannels;
		else if(name.equalsIgnoreCase("amps")) return ampChannels;
		else if(name.equalsIgnoreCase("wafers")) return waferChannels;
		else if(name.equalsIgnoreCase("regions")) return regionChannels;
		else return null;
	}
	
	public int[] getBlindChannels() {
		//return new int[] { 3, 162 };
		return new int[] { 3 }; 
		// pixel 162 has very bad 1/f behaviour, therefore should not be used if possible...
	}

	private Vector2D reuseOffset = new Vector2D();
	public final Vector2D getMapOffsets(int t, int bol) {
		Vector2D mapOffset = reuseOffset;

		if(!map.isHorizontal) map.projection.project(frame[t].getCoords(channel[bol]), mapOffset);
		else mapOffset = frame[t].getdAZdEL(channel[bol]);

		return mapOffset;
	}	
	
	public synchronized void correctForTemperatureDrifts() throws IOException {
		System.err.println("   Correcting for temperature drifts.");
		
		LabocaPixel.readTemperatureGains(instrumentName + File.separator + "he3-gains.dat", channel);
	
		double sum = 0.0;
		int n=0;
		for(int t=0; t<frames; t++) if(frame[t] != null) {
			sum += frame[t].he3Temp;
			n++;
		}
		double aveHe3Temp = sum/n;
		
		double var = 0.0;
		for(int t=0; t<frames; t++) if(frame[t] != null) {
			double dev = frame[t].he3Temp - aveHe3Temp;
			var += dev * dev;
		}
		rmsHe3 = Math.sqrt(var/(n-1));
		
		System.err.println("   RMS He3 temperature drift is " + Util.f3.format(1e3 * rmsHe3) + " mK.");

		// Apply correction
		for(int t=0; t<frames; t++) if(frame[t] != null) {
			double dT = frame[t].he3Temp - aveHe3Temp;
			float[] readout = frame[t].data;
			for(int c=0; c<channels; c++) readout[c] -= channel[c].temperatureGain * dT;
		}
	}
	
	public void getHe3TemperatureGains() throws IOException {
		System.err.println("   Calculating He3 Temperature Gains.");
		
		double sum = 0.0;
		int n=0;
		for(int t=0; t<frames; t++) if(frame[t] != null) {
			sum += frame[t].he3Temp;
			n++;
		}
		double aveHe3Temp = sum/n;
	
		// Level the signals...
		removeMLOffsets(LabocaPixel.getAllChannels(channel, LabocaPixel.FLAG_DEAD), 0, frames);
		
		for(int c=0; c<channels; c++) {
			double sumIT = 0.0, sumT2 = 0.0;
			for(int t=0; t<frames; t++) if(frame[t] != null) {
				double T = frame[t].he3Temp - aveHe3Temp;
				sumIT += frame[t].data[c] * T;
				sumT2 += T * T;
			}
			channel[c].temperatureGain = sumIT / sumT2;
		}
		
		// Now write to a file
		String fileName = MiniCRUSH.workPath + File.separator + "he3-gains-" + serialNo + ".dat";
		try { LabocaPixel.writeTemperatureGains(fileName, channel); }
		catch(IOException e) { e.printStackTrace(); }
		
		System.err.println(" Exiting.");
		
		System.exit(0);
	}

	
	public synchronized void correctViaBlinds() throws IOException {
		int[] blindChannels = getBlindChannels();
		if(blindChannels.length > 0);
		
		System.err.println("   Correcting for temperature drifts (Blind bolometers).");
		
		LabocaPixel.readTemperatureGains(instrumentName + File.separator + "he3-gains.dat", channel);
		
		float[] gain = new float[blindChannels.length];
		for(int k=0; k<blindChannels.length; k++) gain[k] = (float) channel[blindChannels[k]].temperatureGain;
	
		float[] signal = new float[frames];
		float[] smoothed = new float[frames];
		
		
		for(int k=0; k<blindChannels.length; k++) channel[blindChannels[k]].flag = 0;
		removeMLOffsets(blindChannels, 0, frames);
		if(blindChannels.length > 1) getCorrelated(blindChannels, gain, signal, false);
		else {
			int c = blindChannels[0];
			for(int t=0; t<frames; t++) signal[t] = (float) (frame[t].data[c] / channel[c].temperatureGain);
		}
		
		for(int k=0; k<blindChannels.length; k++) channel[blindChannels[k]].flag = LabocaPixel.FLAG_BLIND;
			
		int smoothT = Math.min((int)Math.ceil(blindTimeScale / integrationTime), frames);
		
		double sum = 0.0;
		for(int t=0; t<smoothT-1; t++) sum += signal[t];
		
		for(int fromt=smoothT, tot=smoothT/2; fromt<frames; fromt++, tot++) {
			sum += signal[fromt];
			smoothed[tot] = (float) (sum / smoothT);
			sum -= signal[fromt-smoothT];
		}
		for(int t=0; t<smoothT/2; t++) smoothed[t] = smoothed[smoothT/2];
		for(int t=frames-smoothT/2; t<frames; t++) smoothed[t] = smoothed[frames-smoothT/2];
		
		// Calculate the RMS temperature fluctuation...
		
		sum = 0.0;
		int from = smoothT/2;
		int to = frames - from;
		for(int t=from; t<to; t++) sum += smoothed[t] * smoothed[t];
		rmsHe3 = Math.sqrt(sum / (to-from-1));
		
		System.err.println("   RMS He3 temperature drift is " + Util.f3.format(1e3 * rmsHe3) + " mK.");
		
		
		// Apply correction
		for(int t=0; t<frames; t++) {
			float[] readout = frame[t].data;
			for(int c=0; c<channels; c++) readout[c] -= channel[c].temperatureGain * smoothed[t];
		}
	}	
	
	public void removeOffsets(int[] channelSelection, int fromt, int tot, boolean robust) {
		if(robust) robustRemoveOffsets(channelSelection, fromt, tot);
		else removeMLOffsets(channelSelection, fromt, tot);
	}
		
	private synchronized void removeMLOffsets(final int[] channelSelection, final int fromt, int tot) {
		tot = Math.min(tot, frames);

		//checkForNaNs();
		
		for(int k=0; k<channelSelection.length; k++) {
			final LabocaPixel pixel = channel[channelSelection[k]];
			final int c = pixel.index;
			double offset = 0.0, sumw = 0.0; 

			for(int t=fromt; t<tot; t++) if(frame[t] != null) {
				final LabocaFrame exposure = frame[t];
				exposure.temp = 0.0;
				if((exposure.flag & LabocaFrame.MODELING_FLAGS) == 0) if(exposure.sampleFlag[c] == 0) {
					exposure.temp = exposure.relativeWeight;
					offset += exposure.temp * exposure.data[pixel.index];
					sumw += exposure.temp;
				}
			}

			if(sumw > 0.0) {
				offset /= sumw;
				for(int t=fromt; t<tot; t++) if(frame[t] != null) {
					final LabocaFrame exposure = frame[t];
					exposure.data[c] -= offset;
					exposure.lastDependents += exposure.temp / sumw; 
				}
				pixel.lastDependents++;
			}
		}
		
		if(debug) checkForNaNs();
	}

	
	
	private WeightedPoint[] offsetData;

	private synchronized void robustRemoveOffsets(final int[] channelSelection, final int fromt, int tot) {
		tot = Math.min(tot, frames);

		if(offsetData == null) offsetData = new WeightedPoint[tot-fromt];
		if(offsetData.length < tot-fromt) offsetData = new WeightedPoint[tot-fromt];
		if(offsetData[0] == null) for(int i=0; i<offsetData.length; i++) offsetData[i] = new WeightedPoint();
		
		for(int k=0; k<channelSelection.length; k++) {
			final LabocaPixel pixel = channel[channelSelection[k]];
			final int c = pixel.index;
			int n = 0;
			double sumw = 0.0;

			for(int t=fromt; t<tot; t++) if(frame[t] != null) {
				final LabocaFrame exposure = frame[t];
				exposure.temp = 0.0;
				if((exposure.flag & LabocaFrame.MODELING_FLAGS) == 0) if(exposure.sampleFlag[c] == 0) {
					exposure.temp = exposure.relativeWeight;
					offsetData[n].value = exposure.data[c];
					offsetData[n++].weight = exposure.temp;
					sumw += exposure.temp;
				}
			}

			if(n > 0) {
				final double offset = ArrayUtil.median(offsetData, 0, n);
				for(int t=fromt; t<tot; t++) if(frame[t] != null) {
					final LabocaFrame exposure = frame[t];
					exposure.data[c] -= offset;
					exposure.lastDependents += exposure.temp / sumw;
				}
				pixel.lastDependents++;
			}
		}
		if(debug) checkForNaNs();
	}

	public void removeNeighbours(boolean useGains) {
		comments += "N";
		
		// First find the total neighbour weight norms...
		double[] sumw = new double[channels];
		double[] w = new double[channels];
		double[] g = new double[channels];
		
		for(int c=0; c<channels; c++) {
			w[c] = channel[c].weight;
			g[c] = useGains ? channel[c].gain : 1.0;
		}
		
		for(int c1=0; c1<channels; c1++) if(channel[c1].isUnflagged(LabocaPixel.FLAG_DEAD)) 
			for(LabocaPixel ch2 : channel[c1].neighbours) if(ch2.isUnflagged(LabocaPixel.FLAG_DEAD)) {
				if(ch2.flag == 0) sumw[c1] += w[ch2.index] * g[ch2.index] * g[ch2.index];
				if(channel[c1].flag == 0) sumw[ch2.index] += w[c1] * g[c1] * g[c1];
			}
		
		float[] data = new float[channels];
		for(int t=0; t<frames; t++) {
			LabocaFrame exposure = frame[t];
			if(exposure != null) if((exposure.flag & LabocaFrame.MODELING_FLAGS) == 0) {
				float[] readout = frame[t].data;
				for(int c=0; c<channels; c++) data[c] = readout[c]; 

				for(int c1=0; c1<channels; c1++) if(channel[c1].isUnflagged(LabocaPixel.FLAG_DEAD)) if(exposure.sampleFlag[c1] == 0) {
					for(LabocaPixel ch2 : channel[c1].neighbours) if(ch2.isUnflagged(LabocaPixel.FLAG_DEAD)) if(exposure.sampleFlag[ch2.index] == 0) {
						if(channel[c1].neighbours.size() > 1 && ch2.flag == 0 && sumw[c1] > 0.0) 
							readout[c1] -= w[ch2.index] * g[ch2.index] * data[ch2.index] / sumw[c1];
						if(ch2.neighbours.size() > 1 && channel[c1].flag == 0 && sumw[ch2.index] > 0.0)
							readout[ch2.index] -= w[c1] * g[c1] * data[c1] / sumw[ch2.index];
					}
				}
			}
		}
		
	}
	
	public void getCorrelated(int[] channelSelection, float[] gain, float[] correlated, boolean robust) {
		int n = 0;
		
		final int excludeFlags = ~LabocaPixel.FLAG_BLIND;
		
		for(int k=0; k<channelSelection.length; k++) {
			LabocaPixel pixel = channel[channelSelection[k]];
			if(pixel.isUnflagged(excludeFlags) && pixel.weight > 0.0) n++; 
		}
		
		if(n <= 1) return;
		
		if(robust) getRobustCorrelated(0, frames, channelSelection, gain, correlated);
		else getMLCorrelated(0, frames, channelSelection, gain, correlated);
			
		calcGroupFiltering(channelSelection, gain);
	}
		
	// Get correlated for all frames even those that are no good...
	// But use only channels that are valid, and skip over flagged samples...
	private synchronized void getMLCorrelated(int from, int to, int[] channelSelection, float[] gain, float[] correlated) {
		if(channelSelection.length < 2) return;
		
		final int excludeFlags = ~LabocaPixel.FLAG_BLIND;
		
		for(int t=from; t<to; t++) if(frame[t] != null) if((frame[t].flag & LabocaFrame.MODELING_FLAGS) == 0) {
			final LabocaFrame exposure = frame[t];
			double sum = 0.0, sumwG2 = 0.0;
			
			for(int k=0; k<channelSelection.length; k++) {
				final LabocaPixel pixel = channel[channelSelection[k]];
				final float G = gain[k]; 
				pixel.temp = 0.0;
				
				if(pixel.isUnflagged(excludeFlags)) if(exposure.sampleFlag[pixel.index] == 0) {
					final double wG = exposure.relativeWeight * pixel.weight * G;
					pixel.temp = wG * G;
					sum += wG * exposure.data[pixel.index];
					sumwG2 += pixel.temp;					
				}
			}
			if(sumwG2 > 0.0) {
				final double dC = sum / sumwG2;
				
				correlated[t] += dC;	
				
				for(int k=0; k<channelSelection.length; k++) {
					final LabocaPixel pixel = channel[channelSelection[k]];
					exposure.data[pixel.index] -= gain[k] * dC;
					pixel.lastDependents += pixel.temp / sumwG2;
				}
				
				exposure.lastDependents++;
			}
		}
		if(debug) checkForNaNs();
	}
	
	private WeightedPoint[] corrData;
	private synchronized void getRobustCorrelated(int from, int to, final int[] channelSelection, final float[] gain, final float[] correlated) {
		if(channelSelection.length < 2) return;
		
		int excludeFlags = ~LabocaPixel.FLAG_BLIND;
		
		if(corrData == null) corrData = new WeightedPoint[channelSelection.length];
		else if(corrData.length < channelSelection.length) corrData = new WeightedPoint[channelSelection.length];
		if(corrData[0] == null) for(int i=0; i<corrData.length; i++) corrData[i] = new WeightedPoint();
		
		for(int t=from; t<to; t++) if(frame[t] != null) if((frame[t].flag & LabocaFrame.MODELING_FLAGS) == 0) {
			final LabocaFrame exposure = frame[t];
			int n=0;
			double sumwG2 = 0.0;
			
			for(int k=0; k<channelSelection.length; k++) {
				final LabocaPixel pixel = channel[channelSelection[k]];
				final float G = gain[k];
				pixel.temp = 0.0;
				
				if(pixel.isUnflagged(excludeFlags)) if(exposure.sampleFlag[pixel.index] == 0) {
					pixel.temp = exposure.relativeWeight * pixel.weight * G * G;
					corrData[n].value = exposure.data[pixel.index] / G;
					corrData[n++].weight = pixel.temp;
					sumwG2 += pixel.temp;
				}
			}
			if(n > 0) {
				try {
					final double dC = ArrayUtil.smartMedian(corrData, 0, n, 0.25);
				
					correlated[t] += dC;

					for(int k=0; k<channelSelection.length; k++) {
						final LabocaPixel pixel = channel[channelSelection[k]];
						exposure.data[pixel.index] -= gain[k] * dC;
						pixel.lastDependents += pixel.temp / sumwG2;
					}
					
					exposure.lastDependents++;
					
				} catch(IllegalStateException e) {}
			}
		}
		if(debug) checkForNaNs();
	}
	
	
	public void getGains(int[] channelSelection, float[] gain, 
			float[] correlated, double loLimit, double hiLimit, int flagPattern, boolean sourceSync, int normMethod, boolean robust) {	
		
		if(robust) getRobustGains(0, frames, channelSelection, gain, correlated, loLimit, hiLimit, flagPattern, sourceSync, normMethod);
		else getMLGains(0, frames, channelSelection, gain, correlated, loLimit, hiLimit, flagPattern, sourceSync, normMethod);
	}
	

	
	// Calculate gains for *ALL* channels, even if bad.
	private synchronized void getMLGains(int from, int to, final int[] channelSelection, final float[] gain, 
		final float[] correlated, final double loLimit, final double hiLimit, final int flagPattern, final boolean sourceSync, final int normMethod) {
		if(channelSelection.length < 2) return;

		// override the normalization method...
		//normMethod = 0;
		
		final double[] dG = new double[gain.length];

		for(int k=0; k<channelSelection.length; k++) {
			final LabocaPixel pixel = channel[channelSelection[k]];
			double RC = 0.0, C2 = 0.0;

			for(int t=from; t<to; t++) if(frame[t] != null) {
				final LabocaFrame exposure = frame[t];
				exposure.temp = 0.0;
				
				if((exposure.flag & LabocaFrame.MODELING_FLAGS) == 0) if(exposure.sampleFlag[pixel.index] == 0) {
					final double C = correlated[t];
					final double wC = exposure.relativeWeight * C;
					exposure.temp = wC * C;
					RC += wC * exposure.data[pixel.index];
					C2 += exposure.temp;			
				}
			}
			if(C2 > 0.0) {
				dG[k] = RC / C2;
				for(int t=from; t<to; t++) if(frame[t] != null) {
					final LabocaFrame exposure = frame[t];
					exposure.lastDependents += exposure.temp / C2;
				}
				pixel.lastDependents++;
			}
		}

		// Important!!! Must synch with the maximum likelihood estimates
		// If renormalizing, then correlated has to be derived again before can be injected...
		syncGains(channelSelection, gain, dG, correlated, sourceSync);	
		normGainChanges(channelSelection, gain, dG, hiLimit, loLimit, normMethod, correlated);
		flagGains(channelSelection, gain, loLimit, hiLimit, flagPattern, normMethod); 	
		//syncGains(channelSelection, gain, dG, correlated, sourceSync);			
	}
	
	
	private WeightedPoint[] gainData;
	private synchronized void getRobustGains(int from, int to, final int[] channelSelection, final float[] gain, 
			final float[] correlated, final double loLimit, final double hiLimit, final int flagPattern, final boolean sourceSync, final int normMethod) {
		if(channelSelection.length < 2) return;

		// override the normalization method...
		//normMethod = 0;

		final double[] dG = new double[gain.length];
		
		if(gainData == null) gainData = new WeightedPoint[frames];
		if(gainData.length < frames) gainData = new WeightedPoint[frames];
		
		if(gainData[0] == null) for(int t=0; t<frames; t++) gainData[t] = new WeightedPoint();

		for(int k=0; k<channelSelection.length; k++) {
			final LabocaPixel pixel = channel[channelSelection[k]];
			int n=0;
			double C2 = 0.0;

			for(int t=from; t<to; t++) if(frame[t] != null) {
				final LabocaFrame exposure = frame[t];
				exposure.temp = 0.0;
				if((exposure.flag & LabocaFrame.MODELING_FLAGS) == 0) if(exposure.sampleFlag[pixel.index] == 0) {
					final double C = correlated[t];
					exposure.temp = exposure.relativeWeight * C * C;
					gainData[n].value = exposure.data[pixel.index] / C;
					gainData[n++].weight = exposure.temp;
					C2 += exposure.temp;
				}
			}

			try { 
				dG[k] = n > 0 ? ArrayUtil.smartMedian(gainData, 0, n, 0.25) : 0.0; 
				for(int t=from; t<to; t++) if(frame[t] != null){
					final LabocaFrame exposure = frame[t];
					exposure.lastDependents += exposure.temp / C2;
				}
				pixel.lastDependents++;
			}
			catch(IllegalStateException e) { dG[k] = 0.0; }
		}

		// Important!!! Must synch with the maximum likelihood estimates
		// If renormalizing, then correlated has to be derived again before it can be injected...
		syncGains(channelSelection, gain, dG, correlated, sourceSync);
		normGainChanges(channelSelection, gain, dG, hiLimit, loLimit, normMethod, correlated);
		flagGains(channelSelection, gain, loLimit, hiLimit, flagPattern, normMethod); 
		//syncGains(channelSelection, gain, dG, correlated, sourceSync);

	}
	
	
		
	private synchronized void normGainChanges(int[] channelSelection, float[] gain, double[] dG, double hiLimit, double loLimit, int normMethod, float[] correlated) {
		// Normalize gain change by the average of pixels that see light and have not already been flagged for abnormal gains...
		
		int pixelFlags = LabocaPixel.FLAG_DEAD | LabocaPixel.FLAG_BLIND | LabocaPixel.FLAG_GAIN;
		
		WeightedPoint[] point = new WeightedPoint[channelSelection.length];
		for(int i=0; i<channelSelection.length; i++) point[i] = new WeightedPoint();
		int n=0;
		
		if(normMethod == LabocaPixel.GAINS_POSITIVE) {
			for(int k=0; k<channelSelection.length; k++) {
				LabocaPixel pixel = channel[channelSelection[k]];
				if((pixel.flag & pixelFlags) == 0) {
					point[n].value = dG[k] / gain[k];
					point[n++].weight = pixel.weight * gain[k] * gain[k];
				}
			}
			if(n > 0) {
				double avedG = ArrayUtil.median(point, 0, n);
				for(int k=0; k<channelSelection.length; k++) dG[k] -= avedG;
			}
		}
		else if(normMethod == LabocaPixel.GAINS_BIDIRECTIONAL) {		
			for(int k=0; k<channelSelection.length; k++) {
				LabocaPixel pixel = channel[channelSelection[k]];
				if((pixel.flag & pixelFlags) == 0) {
					point[n].value = Math.abs(gain[k] + dG[k]); 
					point[n++].weight = pixel.weight * gain[k] * gain[k];
				}
			}
			if(n > 0) {
				double norm = ArrayUtil.median(point, 0, n);
				for(int k=0; k<channelSelection.length; k++) dG[k] = (gain[k] + dG[k]) / norm - gain[k];
			}
		}
		
		double sumwg2=0.0, oldsumwg2=0.0;
		for(int k=0; k<channelSelection.length; k++) {
			LabocaPixel pixel = channel[channelSelection[k]];
			if((pixel.flag & pixelFlags) == 0) {
				double G = pixel.gain;
				sumwg2 += pixel.weight * G * G;
				G += dG[k];
				oldsumwg2 += pixel.weight * G * G;
			}
		}
		
		// Keep the sum w g^2 C product the same...
		final double factor = oldsumwg2 / sumwg2;
		for(int t=0; t < correlated.length; t++) correlated[t] *= factor;
		
		// Report if a gain change is unrealistic...
		/*
		for(int k=0; k<channelSelection.length; k++) {
			LabocaPixel pixel = channel[channelSelection[k]];
			if((pixel.flag & pixelFlags) == 0) if(Math.abs(dG[k]) > 1.0) System.err.println(" WARNING! dG[" + (pixel.index+1) + "]: " + Util.f1.format(dG[k]));
		}
		*/
	}
	
	private synchronized void syncGains(int[] channelSelection, float[] gain, double[] dG, float[] correlated, boolean sourceSync) {
		// Sync to all channels in selection.
		for(int k=0; k<channelSelection.length; k++) {
			final LabocaPixel pixel = channel[channelSelection[k]];
			gain[k] += dG[k];	    
			
			// Sync to all frames
			for(int t=0; t<frames; t++) if(frame[t] != null) {
				final LabocaFrame exposure = frame[t];
				exposure.data[pixel.index] -= dG[k] * correlated[t];

				if(sourceSync && sourceGeneration > 0) if((exposure.flag & LabocaFrame.SKIP_SOURCE) == 0) {
					final double frameGain = exposure.transmission; 
					final Vector2D offset = getMapOffsets(t, pixel.index);
					exposure.data[pixel.index] -= dG[k] * frameGain * map.signal[map.indexOfdX(offset.x)][map.indexOfdY(offset.y)];
				}
			}
			pixel.lastDependents++;
		}
	}

	private synchronized void flagGains(int[] channelSelection, float[] gain, double loLimit, double hiLimit, int flagPattern, int normMethod) {
		// Flag/unFlag
		if(flagPattern > 0) for(int k=0; k<channelSelection.length; k++) {
			LabocaPixel pixel = channel[channelSelection[k]];
			
			double G = Double.NaN;
			if(normMethod == LabocaPixel.GAINS_POSITIVE) G = gain[k];
			else if(normMethod == LabocaPixel.GAINS_BIDIRECTIONAL) G = Math.abs(gain[k]);

			if(G < loLimit || G > hiLimit) pixel.flag |= flagPattern;
			else pixel.flag &= ~flagPattern;
		}
		
		if(debug) checkForNaNs();
	}
	
	
	private synchronized void applyGains(int[] channelSelection, float[] channelGain) {
		for(int k=0; k<channelSelection.length; k++) channel[channelSelection[k]].gain = channelGain[k];
	}


	public void census() {
		goodChannelCount = 0;
		for(int c=0; c<channels; c++) if(channel[c].flag == 0 && channel[c].weight > 0.0) goodChannelCount++;
		if(debug) checkForNaNs();

	}

	public void getPixelWeights() {
		comments += "W";

		//checkForNaNs();

		for(int c=0; c<channels; c++) if(channel[c].isUnflagged(LabocaPixel.FLAG_DEAD)) {
			int points = 0;
			double var=0.0;
			
			for(int t=0; t<frames; t++) {
				LabocaFrame exposure = frame[t];
				if(exposure != null) if((exposure.flag & LabocaFrame.WEIGHTING_FLAGS) == 0) if(exposure.sampleFlag[c] == 0) {
					double value = exposure.data[c];
					var += exposure.relativeWeight * value * value;
					points++;
				}
			}
			
			channel[c].dof = 1.0 - channel[c].dependents / points;
			channel[c].weight = channel[c].dof * points / var;
		}

		flagWeights();
		
		census();
	}
	
	public void getSourceWeights(double sourceSize) {	
			
		int nFrames = (int)Math.ceil((map.beamFWHM / aveScanSpeed) / integrationTime);
		double sumpw = 0.0;
		
		comments += "SW";
		
		int nBlocks = (int) Math.ceil((double) frames / nFrames);
	
		for(int c=0; c<channels; c++) if(channel[c].isUnflagged(LabocaPixel.FLAG_DEAD)) {
			LabocaPixel pixel = channel[c];
			int points = 0;
			double var=0.0;
			
			for(int j=0,t=0; j<nBlocks; j++) {
				double sumx = 0.0, sumx2 = 0.0, sumw = 0.0;
				
				for(int blockt = 0; blockt < nFrames && t < frames; blockt++, t++) {
					LabocaFrame exposure = frame[t];
					
					if(exposure != null) if((exposure.flag & LabocaFrame.WEIGHTING_FLAGS) == 0) if(exposure.sampleFlag[c] == 0) {
						double value = exposure.data[c];
						sumx += exposure.relativeWeight * value;
						sumx2 += exposure.relativeWeight * value * value;
						sumw += exposure.relativeWeight;
						points++;
					}
				}

				if(sumw > 0.0) var += sumx2 - sumx * sumx / sumw;
			}

			if(pixel.flag == 0) sumpw += pixel.weight * pixel.sourceFiltering * pixel.sourceFiltering;
			channel[c].sourceWeight = points / var;
		}

		sourceNEFD = Math.sqrt(channels*integrationTime/sumpw)/(Unit.uV * Math.sqrt(Unit.s)) * janskyPerMicroV;

		comments += "{" + Util.e2.format(sourceNEFD) + "}";
		
	}

	public void getDifferencialPixelWeights() {
		comments += "w";
		int delta = 10;

		//checkForNaNs();

		for(int c=0; c<channels; c++) if(channel[c].isUnflagged(LabocaPixel.FLAG_DEAD)) {
			double var=0.0;
			int points = 0;

			for(int t=delta; t<frames; t++) {
				LabocaFrame exposure = frame[t];
				LabocaFrame prior = frame[t-delta];
				if(exposure != null && prior != null) if(((exposure.flag | prior.flag) & LabocaFrame.WEIGHTING_FLAGS) == 0)
					if(exposure.sampleFlag[c] == 0) if(prior.sampleFlag[c] == 0) {
						double value = exposure.data[c] - prior.data[c];
						var += exposure.relativeWeight * value * value;
						points++;
					}		    
			}

			channel[c].dof = 1.0 - channel[c].dependents / points;
			channel[c].weight = 2.0 * channel[c].dof * points / var;			
		}
 
		flagWeights();
		census();
		
	}

	
	public void getRobustPixelWeights() {
		comments += "[W]";
	
		double[] dev2 = new double[frames];

		for(int c=0; c<channels; c++) if(channel[c].isUnflagged(LabocaPixel.FLAG_DEAD)) {
			int points = 0;
			
			for(int t=0; t<frames; t++) {
				LabocaFrame exposure = frame[t];
				if(exposure != null) if((exposure.flag & LabocaFrame.WEIGHTING_FLAGS) == 0) if(exposure.sampleFlag[c] == 0) {
					double value = frame[t].data[c];
					dev2[points++] = exposure.relativeWeight * value * value;
				}		    
			}	
			
			if(points > 0) {
				channel[c].dof = 1.0 - channel[c].dependents / points;
				channel[c].weight = channel[c].dof / (ArrayUtil.median(dev2, 0, points) / 0.454937);
			}
			else {
				channel[c].dof = 0.0;
				channel[c].weight = 0.0;				
			}
		}
		
		flagWeights();
		
		census();
	}
	
	
	// Changed to flag with gains!!!
	public void flagWeights() {
		double min = 0.0;
		double max = Double.POSITIVE_INFINITY;
		
		if(options.containsKey("relative-noise-range")) {
			StringTokenizer tokens = new StringTokenizer(options.get("relative-noise-range"), ",:- \t\r\n");
			if(tokens.hasMoreTokens()) {
				max = 1.0 / Double.parseDouble(tokens.nextToken());
				max *= max;
			}
			if(tokens.hasMoreTokens()) {
				min = 1.0 / Double.parseDouble(tokens.nextToken());
				min *= min;
			}
		}
		
		if(goodChannelCount == 0) {
			comments += "(----)";
			return;			
		}
		
		// Flag channels with insufficient degrees of freedom
		double[] weights = new double[channels];
		int n=0;
		for(int c=0; c<channels; c++) if(channel[c].isUnflagged(LabocaPixel.FLAG_DEAD)) {
	
			if(channel[c].dof > 0.0) {
				channel[c].flag &= ~LabocaPixel.FLAG_DOF;
				if((channel[c].flag & ~LabocaPixel.FLAG_SENSITIVITY) == 0)
					weights[n++] = channel[c].weight * channel[c].gain * channel[c].gain;
			}
			else {
				channel[c].flag |= LabocaPixel.FLAG_DOF;
				channel[c].weight = 0.0;
			}
		}
				
		// Flag out channels with unrealistically small or large weights
		
		double medianWeight = n > 0 ? ArrayUtil.median(weights, 0, n) : 0.0;
		
		if(n == 0) { 
			comments += "(DOF?)";
			return;
		}
		
		double maxWeight = max * medianWeight;
		double minWeight = min * medianWeight;
		
		double sumw = 0.0;

		for(int c=0; c<channels; c++) if(channel[c].isUnflagged(LabocaPixel.FLAG_DEAD)) if(channel[c].weight > 0.0) {
			double wg2 = channel[c].weight * channel[c].gain * channel[c].gain / channel[c].dof;
			
			channel[c].flag &= ~LabocaPixel.FLAG_SENSITIVITY;
			
			if(wg2 > maxWeight) channel[c].flag |= LabocaPixel.FLAG_SENSITIVITY;
			else if(wg2 < minWeight) channel[c].flag |= LabocaPixel.FLAG_SENSITIVITY;		
			else if(channel[c].flag == 0) sumw += wg2;
		}
		if(sumw <= 0.0) { 
			comments += "(NEFD?)";
			return;
		}
		
		
		nefd = Math.sqrt(channels*integrationTime/sumw)/(Unit.uV * Math.sqrt(Unit.s)) * janskyPerMicroV;
		
		census();
		
		comments += "(" + Util.e2.format(nefd) + ")";
		comments += goodChannelCount + " ";
	}

	public void getFrameWeights(int blockSize) {
		comments += "FW";

		//checkForNaNs();

		double sumfw = 0.0;
		int nFrames = 0;
		
		int nT = (int)Math.ceil((float)frames / blockSize);
		
		for(int T=0; T<nT; T++) {
			int points = 0;
			double deps = 0;
			double var=0.0;
			
			final int fromt = T*blockSize;
			final int tot = Math.min(frames, fromt+blockSize);
			
			for(int t=fromt; t < tot; t++) if(frame[t] != null) {
				final LabocaFrame exposure = frame[t];
				
				for(int c=0; c<channels; c++) if(channel[c].isUnflagged(LabocaPixel.FLAG_DEAD)) if(exposure.sampleFlag[c] == 0) {
					double value = exposure.data[c];
					var += channel[c].weight * value * value;
					points++;
				}
				
				deps += exposure.dependents;
			}
				
			if(points >= deps + 1.0 && var > 0.0) {
				final double dof = 1.0 - deps/points;
				final double fw = (points-deps) / var;
			
				for(int t=fromt; t < tot; t++) if(frame[t] != null) {
					final LabocaFrame exposure = frame[t];
					exposure.flag &= ~LabocaFrame.FLAG_DOF;
					exposure.dof = dof;
					exposure.relativeWeight = fw;
					sumfw += fw;
					nFrames++;
				}
			}
			else for(int t=fromt; t < tot; t++) if(frame[t] != null) {
				frame[t].relativeWeight = 0.0;
				frame[t].flag |= LabocaFrame.FLAG_DOF;
			}
		}
		
		double minw = 0.0;
		double maxw = Double.POSITIVE_INFINITY;
		
		if(options.containsKey("time-weight-range")) {
			StringTokenizer tokens = new StringTokenizer(options.get("time-weight-range"), "-,: \t\n\r");
			if(tokens.hasMoreTokens()) minw = Double.parseDouble(tokens.nextToken());
			if(tokens.hasMoreTokens()) maxw = Double.parseDouble(tokens.nextToken());
		}
		
		if(nFrames > 0) {
			final double avew = sumfw / nFrames;
			for(int t=0; t<frames; t++) if(frame[t] != null) {
				final LabocaFrame exposure = frame[t];
				exposure.relativeWeight /= avew;		
				if(exposure.relativeWeight > maxw || exposure.relativeWeight < minw) exposure.flag |= LabocaFrame.FLAG_WEIGHT;
				else exposure.flag &= ~LabocaFrame.FLAG_WEIGHT;
			}
		}
		
		//flagFrameWeights(0.01, 100.0);
		
	}


	public void noDependents() {
		for(int c=0; c<channels; c++) channel[c].dependents = 0.0;
		for(int t=0; t<frames; t++) if(frame[t] != null) frame[t].dependents = 0.0;
		modelParms.clear();
	}
	
	public synchronized void noLastDependents() {
		for(int c=0; c<channels; c++) channel[c].lastDependents = 0.0;
		for(int t=0; t<frames; t++) if(frame[t] != null) frame[t].lastDependents = 0.0;
	}
	
	public synchronized void assignLastDependentsTo(String task) {
		ModelParms model;
	
		if(modelParms.containsKey(task)) model = modelParms.get(task);
		else model = new ModelParms(task, this);

		for(int t=0; t<frames; t++) if(frame[t] != null) {
			frame[t].dependents -= model.forFrame[t];
			model.forFrame[t] += frame[t].lastDependents;
			frame[t].dependents += model.forFrame[t];
		}
			
		for(int c=0; c<channels; c++) {
			channel[c].dependents -= model.forChannel[c];
			model.forChannel[c] += channel[c].lastDependents;
			channel[c].dependents += model.forChannel[c];
		}
				
	}
	
	public void despikeNeighbouring(double significance) {
		comments += "dN";

		for(int c=0; c<channels; c++) {
			double point =Double.NaN;
			if(frame[0] != null && (frame[0].flag & LabocaFrame.MODELING_FLAGS) == 0) 
				if((frame[0].sampleFlag[c] & LabocaFrame.SAMPLE_SOURCE_BLANK) == 0) point = frame[0].data[c];
			
			double next = point;

			double level = significance / Math.sqrt(0.5 * channel[c].weight);

			if(frame[0] != null) frame[0].sampleFlag[c] &= ~LabocaFrame.SAMPLE_SPIKY_NEIGHBOUR;
			
			for(int t=0; t < frames-1; t++) {
				next = Double.NaN;

				LabocaFrame exposure = frame[t];
				LabocaFrame nextFrame = frame[t+1];
				if(nextFrame != null) {		
					nextFrame.sampleFlag[c] &= ~LabocaFrame.SAMPLE_SPIKY_NEIGHBOUR;
					if((nextFrame.flag & LabocaFrame.MODELING_FLAGS) == 0)
						if((nextFrame.sampleFlag[c] & LabocaFrame.SAMPLE_SOURCE_BLANK) == 0) next = nextFrame.data[c];
				}	
					
				
				if(exposure != null) {
					if(Math.abs(point - next) > level / Math.sqrt(exposure.relativeWeight)) {
						exposure.sampleFlag[c] |= LabocaFrame.SAMPLE_SPIKY_NEIGHBOUR;
						nextFrame.sampleFlag[c] |= LabocaFrame.SAMPLE_SPIKY_NEIGHBOUR;
					}
				}

				point = next;
			}
		}

	}

	public void despikeAbsolute(double significance) {
		comments += "dA";

		for(int c=0; c<channels; c++) {
			double level = significance / Math.sqrt(channel[c].weight);

			for(int t=0; t < frames; t++) if(frame[t] != null) {
				final LabocaFrame exposure = frame[t];
				if((exposure.sampleFlag[c] & LabocaFrame.SAMPLE_SOURCE_BLANK) == 0) {
					if(Math.abs(exposure.data[c]) > level / Math.sqrt(exposure.relativeWeight)) 
						exposure.sampleFlag[c] |= LabocaFrame.SAMPLE_SPIKE;
					else
						exposure.sampleFlag[c] &= ~LabocaFrame.SAMPLE_SPIKE;
				}
			}
		}
	}
	
	public void despikeGradual(double significance, double depth) {
		double[] channelRMS = new double[channels];
		
		for(int c=0; c<channels; c++) {
			channelRMS[c] = Math.sqrt(1.0 / channel[c].weight);
		}
			
		comments += "dG";
		
		for(int t=0; t < frames; t++) if(frame[t] != null) if((frame[t].flag & LabocaFrame.MODELING_FLAGS) == 0) {
			LabocaFrame exposure = frame[t];
			
			double maxdev = 0.0;
			// Find the largest not yet flagged as spike deviation.
			for(int c=0; c<channels; c++) if(channel[c].flag == 0) if((exposure.sampleFlag[c] & LabocaFrame.SAMPLE_SOURCE_BLANK) == 0) {
				double dev = Math.abs(exposure.data[c] / channel[c].gain);
				if(dev > maxdev) maxdev = dev;
			}
			if(maxdev > 0.0) {
				double minSignal = depth * maxdev;
				for(int c=0; c<channels; c++) if((exposure.sampleFlag[c] & LabocaFrame.SAMPLE_SOURCE_BLANK) == 0) {
					double level = Math.max(significance * channelRMS[c] / Math.sqrt(exposure.relativeWeight), channel[c].gain * minSignal);
					if(Math.abs(exposure.data[c]) > level) exposure.sampleFlag[c] |= LabocaFrame.SAMPLE_SPIKE;
					else exposure.sampleFlag[c] &= ~LabocaFrame.SAMPLE_SPIKE;
				}
			}
		}
		
	}
	
	public void despikeFeatures(double significance) {
		despikeFeatures(significance, driftN);
	}
	
	public void despikeFeatures(double significance, int maxBlockSize) {
		if(maxBlockSize > frames) maxBlockSize = frames;
		comments += "dF";
		
		WeightedPoint[] timeStream = new WeightedPoint[frames];
		for(int t=0; t<frames; t++) timeStream[t] = new WeightedPoint();		
		
		WeightedPoint diff = new WeightedPoint();
		
		for(int c=0; c<channels; c++) {	
			for(int t=0; t<frames; t++) if(frame[t] != null) {
				LabocaFrame exposure = frame[t];
				WeightedPoint point = timeStream[t];
				point.value = exposure.data[c];
				point.weight = 0.0;
	
				exposure.sampleFlag[c] &= ~LabocaFrame.SAMPLE_SPIKY_FEATURE; // Clear the feature spike flag...
				if((exposure.flag & LabocaFrame.MODELING_FLAGS) == 0) if(exposure.sampleFlag[c] == 0) point.weight = exposure.relativeWeight * channel[c].weight;					
			}
			
			// check and divide...
			int n = frames;
			for(int blockSize = 1; blockSize <= maxBlockSize; blockSize *= 2) {
				for(int T=1; T < n; T++) if(T < n) {
					diff.set(timeStream[T]);
					diff.subtract(timeStream[T-1]);
					if(Math.abs(diff.value) * Math.sqrt(diff.weight) > significance) {
						for(int t=blockSize*(T-1), blockt=0; t<frames && blockt < 2*blockSize; t++, blockt++) {
							LabocaFrame exposure = frame[t];
							if(exposure != null) exposure.sampleFlag[c] |= LabocaFrame.SAMPLE_SPIKY_FEATURE;		
						}
					}
				}
				
				n /= 2;
				
				for(int to=0, from=0; to < n; to++, from += 2) {
					timeStream[to] = timeStream[from];
					timeStream[to].average(timeStream[from+1]);
				}
			}
		}	

	}
	
	public void flagSpikyChannels(int spikeTypes, double flagFraction, int minSpikes, int channelFlag) {
		int maxChannelSpikes = Math.max(minSpikes, (int)Math.round(flagFraction * frames));
		
		// Flag spiky channels even if spikes are in spiky frames
		//int frameFlags = LabocaFrame.MODELING_FLAGS & ~LabocaFrame.FLAG_SPIKY;
		
		// Only flag spiky channels if spikes are not in spiky frames
		int frameFlags = LabocaFrame.MODELING_FLAGS;
		
		for(int c=0; c<channels; c++) {
			channel[c].spikes = 0;
			
			for(int t=0; t<frames; t++) {
				LabocaFrame exposure = frame[t];
				if(exposure != null) if((exposure.flag & frameFlags) == 0) if((exposure.sampleFlag[c] & spikeTypes) != 0) channel[c].spikes++;
			}

			if(channel[c].spikes > maxChannelSpikes) channel[c].flag |= channelFlag;
			else channel[c].flag &= ~channelFlag;
		}
		
		census();
		comments += goodChannelCount + " ";
	}
			
	public void flagSpikyFrames(int spikeTypes, double minSpikes) {
		int spikyFrames = 0;
		
		// Flag spiky frames even if spikes are in spiky channels.
		//int channelFlags = ~(LabocaPixel.FLAG_SPIKY | LabocaPixel.FLAG_FEATURES);
		
		// Flag spiky frames only if spikes are not in spiky channels.
		int channelFlags = ~0;
		
		for(int t=0; t<frames; t++) {
			LabocaFrame exposure = frame[t];

			if(exposure != null) {
				int frameSpikes = 0;

				for(int c=0; c<channels; c++) if(channel[c].isUnflagged(channelFlags))
					if((exposure.sampleFlag[c] & spikeTypes) != 0) frameSpikes++;
				

				if(frameSpikes > minSpikes) {
					exposure.flag |= LabocaFrame.FLAG_SPIKY;
					spikyFrames++;
				}
				else exposure.flag &= ~LabocaFrame.FLAG_SPIKY;
			}
		}
		
		comments += "(" + Util.f1.format(100.0*spikyFrames/frames) + "%)";
	}
	
	public double getSpikyFraction(int spikeTypes) {
		int spikes = 0, points = 0;
		
		for(int t=0; t<frames; t++) {
			LabocaFrame exposure = frame[t];

			if(exposure != null) if((exposure.flag & LabocaFrame.MODELING_FLAGS) == 0) {
				for(int c=0; c<channels; c++) if(channel[c].flag == 0) {
					points++;
					if((exposure.sampleFlag[c] & spikeTypes) != 0) spikes++;
				}
			}
		}
		
		return (float)spikes / points;
	}
	
	
	

	public double[] getSourceGains() {
		calcAverageFiltering();
		
		if(options.containsKey("rcpgains")) {
			for(int c=0; c<channels; c++) {
				final LabocaPixel pixel = channel[c];
				pixel.mainBeamEfficiency = pixel.sourceGain / pixel.gain;	
			}
		}
		else {
			for(int c=0; c<channels; c++) {
				final LabocaPixel pixel = channel[c];
				pixel.sourceGain = pixel.mainBeamEfficiency * pixel.gain;	
			}
		}
		
		final double[] sourceGain = new double[channels];
		for(int c=0; c<channels; c++) {
			final LabocaPixel pixel = channel[c];
			// Assuming that correlated models are all oriented and thus their filtering goes away
			// when the map is iterated...
			sourceGain[c] = pixel.sourceGain * (map.generation == 0 ? pixel.sourceFiltering : pixel.sourceWhitening);
		}
		
		return sourceGain;
	}
	
	
	
	double scanMapPoints = 0;
	
	public SkyMap getSourceIncrement(double sourceSize, double minIntTime, boolean mapFilterCorrection) {
		comments += " [Map] ";
		
		int activeChannels = 0;
		for(int c=0; c<channels; c++) if(channel[c].flag == 0) activeChannels++;
		
		SkyMap scanMap = map.getSkyMap();
		scanMap.reset();
	
		// If there isn't enough good pixels in the scan, do not generate a map...
		// Require at least 50% of the total channels to work...
		if(activeChannels < 0.5*allSkyChannels.length) return scanMap;
	
		if(options.containsKey("sourceweights")) getSourceWeights(sourceSize);
		else for(int c=0; c<channels; c++) channel[c].sourceWeight = channel[c].dof > 0.0 ? channel[c].weight / channel[c].dof : 0.0;
		
		// Only apply filtering corrections at the first source generation...
		if(map.generation == 0) comments += "[C1~" + Util.f2.format(1.0/averageFiltering) + "] ";
		
		final double[] sourceGain = getSourceGains();
		
		int goodFrames = 0;
		
		for(int t=0; t < frames; t++) {
			final LabocaFrame exposure = frame[t];
			if(exposure != null) if((exposure.flag & LabocaFrame.SOURCE_FLAGS) == 0) {
				final double frameGain = gain * exposure.transmission; 
				goodFrames++;

				if(frameGain > 0.0F) for(int k=0; k<allChannels.length; k++) {
					final LabocaPixel pixel = channel[allChannels[k]];
					
					if(pixel.flag == 0 && (exposure.sampleFlag[pixel.index] & LabocaFrame.SAMPLE_SPIKE_FLAGS) == 0)
						scanMap.addPointAt(getMapOffsets(t, pixel.index), 
							exposure.data[pixel.index], 
							sourceGain[pixel.index] * frameGain,
							pixel.sourceWeight * exposure.relativeWeight,
							integrationTime);
				}
			}
		}
		
		scanMap.normalize();
		scanMap.directExposureClip(minIntTime);
		if(sourceGeneration == 0) scanMap.robustLevel();
		
		if(map.generation > 0 && mapFilterCorrection) {
			comments += "[C2=" + Util.f2.format(1.0/averageFiltering) + "] ";
			scanMap.applyCorrection(averageFiltering, map.s2n);
		}

		scanMap.integrationTime += goodFrames * integrationTime;
		scanMapPoints = scanMap.countPoints();
		
		return scanMap;

	}
	
	/*
	public void addSource() {
	calcSourceGains();
		
	for(int t=0; t < frames; t++) if(frame[t] != null) if(!frame[t].flagged) {
	    final float[] residual = frame[t].data;
	    final double frameGain = gain * frame[t].transmission; 
	
	    for(int bol=0; bol < channels; bol++) if(!LabocaPixel.isBad[bol]) {
		final double g = channel[bol].sourceGain * frameGain * channel[bol].sourceFiltering;
		final short[] index = getMapIndexes(t, bol);
		if(reduced.flag[index[0]][index[1]] == 0)
		    residual[bol] += g * (reduced.clean[index[0]][index[1]]);
	    }		    
	}
	}
	 */
	
	

	public synchronized void removeSource(SkyMap source, double[][] old, double blankingLevel) {
		double[] sourceGain = getSourceGains();		
		
		for(int t=0; t < frames; t++) {
			final LabocaFrame exposure = frame[t];
			if(exposure != null) if((exposure.flag & LabocaFrame.SKIP_SOURCE) == 0) {
				final float[] residual = frame[t].data;
				final double frameGain = frame[t].transmission; 

				// Remoe source from all but the blind channels...
				for(int bol=0; bol < channels; bol++) if((channel[bol].flag & LabocaPixel.FLAG_BLIND) == 0) {
					final double G = sourceGain[bol] * frameGain;
					final Vector2D offset = getMapOffsets(t, bol);
					final int i = map.indexOfdX(offset.x);
					final int j = map.indexOfdY(offset.y);

					// Do not check for flags, to get a true difference image...
					residual[bol] -= (float) G * (source.signal[i][j] - old[i][j]);
					
					// Do the blanking here...
					if(source.s2n[i][j] > blankingLevel) exposure.sampleFlag[bol] |= LabocaFrame.SAMPLE_SOURCE_BLANK;
					else exposure.sampleFlag[bol] &= ~LabocaFrame.SAMPLE_SOURCE_BLANK;
				}	    
			}
		}
		
		
		double sumpw = 0.0;
		for(int k=0; k<allChannels.length; k++) if(channel[allChannels[k]].flag == 0) {
			LabocaPixel pixel = channel[allChannels[k]];
			sumpw += pixel.weight * pixel.sourceGain * pixel.sourceGain;
		}
		
		double sumfw = 0.0;
		for(int t=0; t<frames; t++) if(frame[t] != null) {
			LabocaFrame exposure = frame[t];
			if((exposure.flag & LabocaFrame.SOURCE_FLAGS) == 0)
				sumfw += exposure.relativeWeight * exposure.transmission;
		}
	
		double N = Math.min(scanMapPoints, source.countPoints()) / source.pointsPerBeam;
		
		double np = sumpw > 0.0 ? N / sumpw : 0.0;
		double nf = sumfw > 0 ? N / sumfw : 0.0;
		
		for(int k=0; k<allChannels.length; k++) {
			LabocaPixel pixel = channel[allChannels[k]];
			double G = sourceGain[pixel.index];
			if(pixel.flag == 0) pixel.lastDependents += np * pixel.weight * G * G;
		}
		
		for(int t=0; t<frames; t++) if(frame[t] != null) {
			final LabocaFrame exposure = frame[t];
			if((exposure.flag & LabocaFrame.SOURCE_FLAGS) == 0) exposure.lastDependents += nf * exposure.relativeWeight * exposure.transmission;
		}
		
		sourceGeneration++;
	}

	public void blankSource(Vector2D center, double flagRadius, double onSourceRadius) {
		blankSource(new Vector2D[] { center }, flagRadius, onSourceRadius);	
	}
	
	public void blankSource(Vector2D[] center, double flagRadius, double onSourceRadius) {
		//int lastSourcePixel = -1;
		
		final int excludeFlags = LabocaPixel.FLAG_DEAD | LabocaPixel.FLAG_BLIND;
		
		for(int t=0; t < frames; t++) if(frame[t] != null) {
			for(int bol=0; bol < channels; bol++) if(channel[bol].isUnflagged(excludeFlags)) {
				Vector2D offset = frame[t].getdAZdEL(channel[bol]);
				for(int i=0; i<center.length; i++) {
					if(offset.distanceTo(center[i]) <= flagRadius) {
						if(offset.distanceTo(center[i]) <= onSourceRadius) 
							frame[t].sampleFlag[bol] |= LabocaFrame.SAMPLE_SOURCE_BLANK;
						else 
							frame[t].sampleFlag[bol] |= LabocaFrame.SAMPLE_SKIP;
					}
					/*
					if(frame[t].flag == 0) if(channel[bol].flag == 0) if(bol != lastSourcePixel) {
						if(offset.length() < radius) System.err.println("Found source in pixel " + (channel[bol].backendIndex + 1));
						lastSourcePixel = bol;
					}
					*/
				}
			}
		}	    
	}

	
	public WeightedPoint[] getSourceFluxes(Vector2D[] center, double radius) {
		double r2 = radius * radius;
		WeightedPoint[] fluxes = new WeightedPoint[center.length];
		for(int i=0; i<fluxes.length; i++) fluxes[i] = new WeightedPoint();
		
		int excludeFlags = LabocaPixel.FLAG_DEAD | LabocaPixel.FLAG_BLIND;
		
		for(int t=0; t < frames; t++) if(frame[t] != null) if((frame[t].flag & LabocaFrame.SOURCE_FLAGS) == 0) {
			for(int bol=0; bol < channels; bol++) if(channel[bol].isUnflagged(excludeFlags)) {
				final Vector2D offset = frame[t].getdAZdEL(channel[bol]);
				for(int i=0; i<center.length; i++) {
					offset.subtract(center[i]);
					if(offset.norm() <= r2) {
						int x = map.indexOfdX(offset.x);
						int y = map.indexOfdY(offset.y);
						double w = map.weight[x][y];
						
						fluxes[i].value += w * map.signal[x][y];
						fluxes[i].weight += w;						
					}
				}
			}	    
		}
		
		for(int i=0; i<fluxes.length; i++) fluxes[i].value /= fluxes[i].weight;
		
		return fluxes;
	}


	/*
    public void addSource() {	
	for(int t=0; t < frames; t++) if(frame[t] != null) if(!frame[t].flagged) {
	    final float[] residual = frame[t].data;
	    final double frameGain = frame[t].transmission; 

	    for(int bol=0; bol < channels; bol++) if(!LabocaPixel.isBad[bol]) {
		final double g = channel[bol].mainBeamEfficiency * channel[bol].gain * frameGain;
		final short[] index = getMapIndexes(t, bol);
		if(reduced.flag[index[0]][index[1]] == 0)
		    residual[bol] += g * (reduced.clean[index[0]][index[1]]);
	    }		    
	}
    }
	 */

	
	public int getNodPhase() { return onPhase == 1 ? 1 : -1; }

	/*
	public void flagChoppingPixels() {
		double halfBeam = 9.0 * Unit.arcsec;
		
		Vector2D on = new Vector2D();
		Vector2D left = new Vector2D(-2.0 * chopperThrow, 0.0);
		Vector2D right = new Vector2D(2.0 * chopperThrow, 0.0);
		
		for(int c=0; c<channels; c++) {
			Vector2D offset = channel[c].position;
			if(offset.distanceTo(on) < halfBeam) channel[c].flag |= LabocaPixel.FLAG_SKIP;
			else if(offset.distanceTo(left) < halfBeam) channel[c].flag |= LabocaPixel.FLAG_SKIP;
			else if(offset.distanceTo(right) < halfBeam) channel[c].flag |= LabocaPixel.FLAG_SKIP;
		}
		
		for(int c=0; c<channels; c++) if((channel[c].flag & LabocaPixel.FLAG_SKIP) != 0) {
			System.err.println(" Chopping on pixel " + (channel[c].backendIndex + 1));
			for(int t=0; t<frames; t++) if(frame[t] != null) frame[t].sampleFlag[c] |= LabocaFrame.SAMPLE_SOURCE_BLANK;
		}
	}
	*/
	
	public void markChopped() {
		// Flag pixels that chop on source
		// left and right are the pixel positions, where, if there's a pixel, it will have the source
		// in the left or right beams...
		//System.err.println("on phase is " + subscan[i][k].onPhase);
		Vector2D left = new Vector2D(onPhase == 1 ? 0.0 : 2.0 * chopperThrow, 0.0);
		Vector2D right = new Vector2D(onPhase == 1 ? -2.0 * chopperThrow : 0.0, 0.0);

		for(int c=0; c<channels; c++) {
			LabocaPixel pixel = channel[c];
			if(pixel.position.distanceTo(left) < beamFWHM / 2.0) pixel.flag |= LabocaPixel.CHOPPED_LEFT_BEAM;
			else if(pixel.position.distanceTo(right) < beamFWHM / 2.0) pixel.flag |= LabocaPixel.CHOPPED_RIGHT_BEAM;
			if((pixel.flag & LabocaPixel.CHOPPED_BEAM) != 0) 
				System.err.println("   Source is chopped on pixel " + (pixel.backendIndex + 1));
		}
	}	

	
	public void centerChopped(double tolerance, boolean isRobust) {
		comments += "o";
		
		int fromt = 0;
		int n=0;
		
		while(fromt < frames) {
			fromt = centerChopped(fromt, tolerance, isRobust);	
			n++;
		}
		
		//System.err.println(n + " chopping cycles found.");
	}
	
	// Find center: (aveleft+averight)/2
	public synchronized int centerChopped(int fromt, double tolerance, boolean isRobust) {
		
		double[] left = new double[channels];
		double[] right = new double[channels];
		int nLeft=0, nRight=0;
		
		
		int sign = Math.abs(frame[fromt].chopperPosition) - tolerance > 0.0 ? (int)Math.signum(frame[fromt].chopperPosition) : 0; 
		boolean transitioned = false;
		boolean completedCycle = false;
		
		if(sign < 0) {
			for(int c=0; c<channels; c++) left[c] += frame[fromt].data[c];
			nLeft++;
		}
		else if(sign > 0) {
			for(int c=0; c<channels; c++) right[c] += frame[fromt].data[c];
			nRight++;
		}
		
		
		int tot=fromt;
		while(tot < frames && !completedCycle) {
			while(frame[tot] == null && tot < frames-1) tot++;
			int nextSign = Math.abs(frame[tot].chopperPosition) - tolerance > 0.0 ? (int)Math.signum(frame[tot].chopperPosition) : 0; 
			boolean isTransition = sign * nextSign == -1;
			if(isTransition) {
				if(!transitioned) transitioned = true;
				else completedCycle = true;
			}
			if(nextSign != 0) sign = nextSign;
			
			if(sign < 0) {
				for(int c=0; c<channels; c++) left[c] += frame[tot].data[c];
				nLeft++;
			}
			else if(sign > 0) {
				for(int c=0; c<channels; c++) right[c] += frame[tot].data[c];
				nRight++;
			}
			
			tot++;
		}
		
		if(!completedCycle) {
			for(int t=fromt; t<tot; t++) if(frame[t] != null) frame[t].flag |= LabocaFrame.BAD_DATA;
			return tot;
		}
		
		for(int c=0; c<channels; c++) {
			double value = (left[c]/nLeft + right[c]/nRight)/2.0;
			for(int t=fromt; t<tot; t++) if(frame[t] != null) frame[t].data[c] -= value;
		}
		
		return tot;		
	}

	public WeightedPoint getPhotometry() {
		getSourceGains();
		double sum = 0.0, sumw = 0.0;
		int n = 0;
		for(int t=0; t<frames; t++) {
			LabocaFrame exposure = frame[t];
			if(exposure != null) if((exposure.flag & LabocaFrame.SOURCE_FLAGS) == 0)
				for(int c=0; c<channels; c++) if(channel[c].flag == 0) if(exposure.sampleFlag[c] == LabocaFrame.SAMPLE_SOURCE_BLANK) {
					LabocaPixel pixel = channel[c];
					double G = gain * pixel.sourceGain * exposure.transmission;	
					sum += pixel.weight * G * exposure.data[c]; 
					sumw += pixel.weight * G * G;
					n++;
				}
		}
		
		return new WeightedPoint(sum/sumw, sumw);
	}
	
	public void whitenTo(double level, double sourceSize) {
		whitenTo(ArrayUtil.getPaddedFFTSize(driftN), level, sourceSize, false);
	}
		
	public synchronized void whitenTo(int windowSize, double level, double sourceSize, boolean outsideOnly) {
	comments += "[whiten:";
		
		windowSize = ArrayUtil.getPaddedFFTSize(windowSize);
		// convert critical whitening power level to amplitude...
		level = Math.sqrt(level);
		
		if(frames < windowSize) {
			System.err.println("Whitening window too large.");
			return; 
		}

		final int n = ArrayUtil.getPaddedFFTSize(frames);
		final Complex[] spectrum = new Complex[n/2+1];
		final double[] data = new double[n];
		final int stretch = n/windowSize;
		
		for(int f=0; f<spectrum.length; f++) spectrum[f] = new Complex();
		
		final int nF = windowSize / 2;
		final double[] A = new double[nF];
		final double[] temp = new double[nF];
		final double[] scaling = new double[nF];
		final Complex[] sourceSpectrum = new Complex[nF+1];
		for(int F=0; F<sourceSpectrum.length; F++) sourceSpectrum[F] = new Complex(); 
		final double[] sourceProfile = new double[2*nF];
		
		// TODO
		// to calculate source filtering, assume a Gaussian source with average crossing times...
		//
		// sigma_f sigma_t = 1/2pi
		// sigma_f T = 2.35/2pi
		// A(f) ~ exp(-f^2/2sigma_f^2)
		// P(f) ~ exp(-f^2/sigma_f^2);

		double crossingTime = Math.hypot(map.beamFWHM, sourceSize) / aveScanSpeed;
		double sigma_f = 2.35/(2.0 * Math.PI * crossingTime);
		double dF = 1.0 / (windowSize * integrationTime);
		double sigma_F = sigma_f / dF;
		final int minF = outsideOnly ? (int)Math.floor(1.17 * sigma_F / Math.sqrt(2.0)) : 0;  
		
		// Calculate the typical source profile (beam smoothed!)
		final double[] source = new double[nF];
		double sumSource = 0.0;
		for(int F=0; F<scaling.length; F++) {
			final double d = F / sigma_F;
			source[F] = Math.exp(-0.5 * d*d);
			sumSource += source[F];
		}
		
		// TODO For Gaussian sources it should be possible to calculate this analytically...
		for(int F=0; F<nF; F++) sourceSpectrum[F].set(source[F], 0.0);
		sourceSpectrum[nF].set(0.0, 0.0);
		ArrayUtil.uncheckedBackFFT(sourceSpectrum, sourceProfile);
		final double sourcePeak = sourceProfile[0];
		//for(int t=1; t<sourceProfile.length; t++) if(sourceProfile[t] > sourcePeak) sourcePeak = sourceProfile[t];
		
		//int pixelFlags = LabocaPixel.FLAG_DEAD | LabocaPixel.FLAG_BLIND | LabocaPixel.FLAG_DISCONTINUITY | LabocaPixel.FLAG_GAIN;
		
		double sumpw = 0.0, aveSourceWhitening = 0.0;
		
		for(int c=0; c<channels; c++) {		
			final LabocaPixel pixel = channel[c];
			
			// If the filterResponse array does not exist, create it...
			if(pixel.filterResponse == null) {
				pixel.filterResponse = new float[nF];
				Arrays.fill(pixel.filterResponse, 1.0F);
			}
			
			// Put the time-stream data into an array, and FFT it...
			// Make sure the rest of the reused time-series array contains not leftover garbage from previous round...
			Arrays.fill(data, 0.0);
			for(int t=0; t<frames; t++) if(frame[t] != null) {
				final LabocaFrame exposure = frame[t];
				if((exposure.flag & LabocaFrame.MODELING_FLAGS) == 0) if(exposure.sampleFlag[c] == 0) data[t] = exposure.relativeWeight * exposure.data[c];
			}
			ArrayUtil.uncheckedFFT(data, spectrum);
			
			// Average power in windows, then convert to rms amplitude
			Arrays.fill(A, 0.0);		
			for(int f=1; f<spectrum.length; f++) A[(f-1)/stretch] += spectrum[f].norm();
			for(int F=0; F<A.length; F++) A[F] = Math.sqrt(A[F] / stretch);
			
			// Save the amplitudes for later use...
			System.arraycopy(A, 0, temp, 0, A.length);
			final double aveA = ArrayUtil.median(temp);
			final double sigmaA = aveA / Math.sqrt(stretch);
			final double target = aveA * level;
			final double critical = target + 2.0 * sigmaA;
			
			// Only whiten those frequencies which have a significant excess power
			// when compared to the specified level over the median spectral power.
				
			Arrays.fill(scaling, 1.0); // To be safe initialize the scaling array here...
		
			// This is the whitening filter...
			for(int F=minF; F<scaling.length; F++) {
				// Check if there is excess power that needs filtering
				if(A[F] > critical) scaling[F] = target / A[F];
				// If it was filtered prior, see if the filtering can be relaxed (not to overdo it...)
				else if(pixel.filterResponse[F] < 1.0) if(A[F] < aveA) scaling[F] = target / A[F];
				
				A[F] *= scaling[F];
			}
		
			// Do a neighbour based round as well, with different resolutions
			// (like a feature seek).
			int N = A.length;
			int maxBlock = N / 4;
			double uncertainty = Math.sqrt(2.0 / stretch);
			
			for(int blockSize = 1; blockSize <= maxBlock; blockSize *= 2) {		
				for(int F = 1; F < N-1; F++) {
					double maxA = Math.max(A[F-1], A[F+1]);
					if(A[F] > maxA * (1.0 + 2.0 * uncertainty)) {
						final double rescale = maxA / A[F];
						for(int blockf = 0, f = F*blockSize; blockf < blockSize; blockf++, f++) scaling[f] *= rescale;
						A[F] = maxA;
					}
				}				
				
				for(int F = 0; F < N-1; F += 2) A[F>>1] = 0.5 * Math.hypot(A[F], A[F+1]);
				N >>= 1;
				
				uncertainty /= Math.sqrt(2.0);
			}		
			
			// Renormalize the whitening scaling s.t. it is median-power neutral
			for(int F=0; F<scaling.length; F++) temp[F] *= scaling[F];
			double norm = aveA / ArrayUtil.median(temp);
			
			double sumPreserved = 0.0;	
			for(int F=0; F<scaling.length; F++) {
				scaling[F] *= norm;
				pixel.filterResponse[F] *= scaling[F];				
				final double phi2 = pixel.filterResponse[F] * pixel.filterResponse[F];
				sumPreserved += phi2;
			}

			double noiseWhitening = Math.sqrt(sumPreserved / scaling.length);		
			pixel.weight *= pixel.noiseWhitening / noiseWhitening;
			pixel.noiseWhitening = noiseWhitening;

			// Figure out how much filtering effect there is on the point source peaks...
			for(int F=0; F<nF; F++) sourceSpectrum[F].set(pixel.filterResponse[F] * source[F], 0.0);
			sourceSpectrum[nF].zero();
			
			ArrayUtil.uncheckedBackFFT(sourceSpectrum, sourceProfile);
			double max = sourceProfile[0];
			//for(int t=1; t<sourceProfile.length; t++) if(sourceProfile[t] > max) max = sourceProfile[t];
			
			if(pixel.sourceWhitening > 0.0) pixel.sourceFiltering /= pixel.sourceWhitening;
			pixel.sourceWhitening = max / sourcePeak;		
			pixel.sourceFiltering *= pixel.sourceWhitening;
			
			if(pixel.flag == 0) {
				aveSourceWhitening += pixel.weight * pixel.sourceWhitening * pixel.sourceWhitening;
				sumpw += pixel.weight;
			}
			
			for(int f=1; f<spectrum.length; f++) spectrum[f].scale(1.0 - scaling[(f-1)/stretch]);
			ArrayUtil.uncheckedBackFFT(spectrum, data); 
			for(int t=0; t<frames; t++) if(frame[t] != null) frame[t].data[c] -= data[t];
	
		}
		
		comments += Util.f2.format(Math.sqrt(aveSourceWhitening/sumpw)) + "]";
		
		
		//flagWeights();		
	}
	
	// 2*bins complex gains per pixel...
	public void multiResonanceClean(int[] channelSelection, double[] signal, Complex[][] gain, int bins) {
		Complex[][] spectrum = new Complex[channelSelection.length][];
		
		// create average spectra for all channels
		// weighted average in phase together (with prior gains if available)
		// the 'signal' is averaged power signal
		// Complex gains by comparing 'signal' to the individual spectra
		// remove filtered periodic patches from timeStream of each channel...
	}
	

	public String getFileName(String path, int scanNumber, String projectID) {		
		File root = new File(path);
		String[] fileName = root.list();
		for(int i=0; i<fileName.length; i++) {
			if(fileName[i].startsWith(projectID)) 
				return getFileName(path + File.separator + fileName[i], scanNumber, projectID);
			if(fileName[i].startsWith("APEX-" + scanNumber)) {
				if(fileName[i].endsWith(".gz")) fileName[i] = fileName[i].substring(0, fileName[i].length()-3);
				if(fileName[i].endsWith(".fits")) fileName[i] = fileName[i].substring(0, fileName[i].length()-5);
				if(fileName[i].endsWith(projectID))
					return path + File.separator + fileName[i];
			}
		}
		return null;
	}
	

	public static LabocaScan unite(LabocaScan[] subscans) {
		LabocaScan scan = (LabocaScan) subscans[0].clone();
		
		Vector<LabocaFrame> buffer = new Vector<LabocaFrame>();
		for(int i=0; i<subscans.length; i++) {
			LabocaScan subscan = subscans[i];
			subscan.trim();

			if(i>0) {
				double mjdGap = subscan.frame[0].MJD - subscans[i-1].frame[subscans[i-1].frames-1].MJD;
				int padding = (int) Math.round(mjdGap * Unit.day / scan.integrationTime);
				if(padding < 0) padding = 0;
				for(int t=0; t<padding; t++) buffer.add(null);
			}

			for(int t=0; t<subscan.frames; t++) buffer.add(subscan.frame[t]);

		}

		scan.frames = buffer.size();
		scan.frame = buffer.toArray(new LabocaFrame[scan.frames]);
		
		return scan;
	}
	
	public LabocaScan readAsOne(MiniCRUSH crush, int scanNumber, String projectID, boolean gzipped) throws IOException, FitsException, HeaderCardException {
		return unite(read(crush, scanNumber, projectID, gzipped));
	}
		
	public LabocaScan[] read(MiniCRUSH crush, int scanNumber, String projectID, boolean gzipped) throws IOException, FitsException, HeaderCardException {
		LabocaScan common = (LabocaScan) clone();
		common.options = crush.options;
		common.serialNo = scanNumber;
		common.isCompressed = gzipped;
		//common.descriptor = "APEX-" + scanNumber + "-" + projectID

		System.err.println();
		System.err.println("Scan " + scanNumber + " (details follow...)");

		//System.err.println("Looking for .fits" + (gzipped ? ".gz" : " files") + "\n   with project ID " + projectID + "\n   in " + path);
		String name = getFileName(path, scanNumber, projectID);
		String ext = ".fits" + (gzipped ? ".gz" : "");
		
		File file = new File(name);
		if(file.isDirectory()) {	
			//System.err.println("Found scan directory " + name); 		
			return readScanDirectory(common, name, ext);
		}
		else {
			//System.err.println("Found scan " + name);
			return readScan(common, name + ext);
		}
	}
	

	
	public LabocaScan[] readScanDirectory(LabocaScan common, String dir, String ext) throws IOException, FitsException, HeaderCardException {
		int subscans = common.readScanInfo(dir + File.separator + "SCAN" + ext);
		common.readPar(dir + File.separator + "LABOCA-ABBA-FEBEPAR" + ext);

		Vector<LabocaScan> buffer = new Vector<LabocaScan>();

		for(int i=0; i<subscans; i++) {
			try {
				LabocaScan subscan = (LabocaScan) common.copy();

				System.err.println(" Subscan {" + (i+1) + "}:");
				subscan.subscanNo = i;
				subscan.readDataPar(dir + File.separator + (i+1) + File.separator + "LABOCA-ABBA-DATAPAR" + ext);
				subscan.readData(dir + File.separator + (i+1) + File.separator + "LABOCA-ABBA-ARRAYDATA-1" + ext);
				subscan.readMonitor(dir + File.separator + (i+1) + File.separator + "MONITOR" + ext);
				subscan.validate();

				buffer.add(subscan);
			}
			catch(Exception e) {
				System.err.println(" ERROR: " + e.getMessage());
				e.printStackTrace();
			}
		}

		LabocaScan[] set = new LabocaScan[buffer.size()];
		for(int i=0; i<buffer.size(); i++) {
			set[i] = buffer.get(i);
			set[i].scanPath = dir;
		}
		
		return set;
	}
	
	public LabocaScan[] readScan(LabocaScan common, String fileName) throws IOException, FitsException, HeaderCardException {
		Fits fits = new Fits(new File(fileName), fileName.endsWith(".gz"));	
		BasicHDU[] hdu = fits.read();
		
		int subscans = common.readScanInfo((BinaryTableHDU) hdu[1]);
		common.readPar((BinaryTableHDU) hdu[2]);

		Vector<LabocaScan> buffer = new Vector<LabocaScan>();

		int k=3;
		for(int i=0; i<subscans; i++) {
			try {
				LabocaScan subscan = (LabocaScan) common.copy();
				subscan.subscanNo = i;
				
				System.err.println(" Subscan {" + (i+1) + "}:");
				
				// HDUs for each subscan can come in any order, so check EXTNAME...
				for(int m=0; m<3; m++, k++) {
					BinaryTableHDU table = (BinaryTableHDU) hdu[k];
					String extName = table.getHeader().getStringValue("EXTNAME");
					
					if(extName.equalsIgnoreCase("DATAPAR-MBFITS")) subscan.readDataPar(table);
					else if(extName.equalsIgnoreCase("ARRAYDATA-MBFITS")) subscan.readData(table);
					else if(extName.equalsIgnoreCase("MONITOR-MBFITS")) subscan.readMonitor(table);
				}
				
				subscan.validate();

				buffer.add(subscan);
			}
			catch(Exception e) {
				System.err.println(" ERROR: " + e.getMessage());
				e.printStackTrace();
			}
		}

		LabocaScan[] set = new LabocaScan[buffer.size()];
		for(int i=0; i<buffer.size(); i++) set[i] = buffer.get(i);

		return set;
	}
	

	// returns the number of subscans.
	public int readScanInfo(String fileName) throws IOException, FitsException, HeaderCardException {
		Fits fits = new Fits(new File(fileName), fileName.endsWith(".gz"));	

		BinaryTableHDU hdu = (BinaryTableHDU) fits.getHDU(1);
		return readScanInfo(hdu);
	}
	
	public int readScanInfo(BinaryTableHDU hdu) throws IOException, FitsException, HeaderCardException {
		Header header = hdu.getHeader();
		// Read in the full HDU data
		//hdu.getData().getData();
		
		equatorial = new EquatorialCoordinates(header.getDoubleValue("CRVAL1") * Unit.deg, header.getDoubleValue("CRVAL2") * Unit.deg, "J2000");
		sourceName = header.getStringValue("OBJECT");
		timeStamp = header.getStringValue("DATE-OBS");
		
		System.err.println(" Source [" + sourceName + "] observed on " + timeStamp.substring(0, timeStamp.indexOf('T')));
		
		if(sourceName == null) sourceName = "Undefined";
		LabocaFrame.longitude = header.getDoubleValue("SITELONG") * Unit.deg;
		LabocaFrame.latitude = header.getDoubleValue("SITELAT") * Unit.deg;

		//isChopped = header.getBooleanValue("WOBUSED");
		chopperThrow = header.getDoubleValue("WOBTHROW") * Unit.deg;
		
		isNativeEquatorial = header.getStringValue("CTYPE1").startsWith("RA");

		return header.getIntValue("NOBS");	
	}

	public void readPar(String fileName) throws IOException, FitsException, HeaderCardException {
		Fits fits = new Fits(new File(fileName), fileName.endsWith(".gz"));

		BinaryTableHDU hdu = (BinaryTableHDU) fits.getHDU(1);
		readPar(hdu);
	}
	
	public void readPar(BinaryTableHDU hdu) throws IOException, FitsException, HeaderCardException {
		Header header = hdu.getHeader();

		instrumentGain = 270.0 * (1<<(int)header.getDoubleValue("FEGAIN"));
		//System.err.println(" Frontend Gain is " + G);

		Object[] row = hdu.getRow(0);

		double[] xOffset = (double[]) row[5];
		double[] yOffset = (double[]) row[6];
		referenceChannel = ((int[]) row[7])[0]-1;

		System.err.println(" Reference pixel is " + (referenceChannel+1));

		channels = xOffset.length;
		channel = new LabocaPixel[channels];

		System.err.println(" " + channels + " channels found.");

		// LabocaPixel.offset: get BOLDCOFF

		for(int c=0; c<channels; c++) {
			final LabocaPixel pixel = (LabocaPixel) channelTemplate.clone();
			pixel.index = c;
			pixel.backendIndex = c;
			pixel.position = new Vector2D(xOffset[c] * Unit.deg, yOffset[c] * Unit.deg);
			pixel.position = new Vector2D();
			channel[c] = pixel;
		}
		
		LabocaPixel.setReference(channel, channel[referenceChannel]);

		// Needed only for pre Feb2007 Files with integer format.
		//G *= 10.0/ ((float[]) row[17])[0] / (1<<15-1);	
	}

	boolean isCompressed = false;
	
	public void readData(String fileName) throws IOException, FitsException, HeaderCardException {
		Fits fits = new Fits(new File(fileName), fileName.endsWith(".gz"));

		BinaryTableHDU hdu =(BinaryTableHDU) fits.getHDU(1);
		readData(hdu);
	}
	
	public synchronized void readData(BinaryTableHDU hdu) throws IOException, FitsException, HeaderCardException {
		//Header header = hdu.getHeader();

		// Read in the full HDU data
		hdu.getData().getData();
		
		Vector<LabocaFrame> buffer = new Vector<LabocaFrame>();
		double lastMJD = Double.NaN;

		for(int t=0; t<frames; t++) {
			final LabocaFrame exposure = frame[t];
			final Object[] row = hdu.getRow(t);
			final double MJD = ((double[]) row[0])[0] * Unit.day;

			if(!Double.isNaN(lastMJD)) {
				final int n = (int)Math.round((MJD - lastMJD) / integrationTime);
				if(n > 50) {
					System.err.println("   Inserting " + n + " empty frames...");
					for(int i=0; i<n; i++) buffer.add(null);
				}
			}

			lastMJD = MJD;

			final float[][] data = (float[][]) row[1];
			exposure.data = new float[data.length];

			for(int c=0; c<data.length; c++) exposure.data[c] = data[c][0];

			exposure.calc();

			buffer.add(exposure);
		}	
		
		for(int c=0; c<channels; c++) {
			channel[c].lastDependents = 0;
			channel[c].spikes = 0;
		}

		frame = new LabocaFrame[frames];
		for(int t=0; t<frames; t++) frame[t] = buffer.get(t);

		detectorStage();
	}
	
	public void writeData(String fromName, String toName) throws IOException, FitsException, HeaderCardException {
		Fits fits = new Fits(new File(fromName), fromName.endsWith(".gz"));
		BinaryTableHDU hdu = (BinaryTableHDU) fits.getHDU(1);
			
		for(int t=0; t<frames; t++) {
			LabocaFrame exposure = frame[t];
			final Object[] row = hdu.getRow(t);
			final float[][] data = (float[][]) row[1];
			for(int c=0; c<data.length; c++) data[c][0] = 0.0F;
			for(int c=0; c<channels; c++) data[channel[c].backendIndex][0] = exposure.data[c];
			hdu.setRow(t, row);
		}	
			
		fits.write(new BufferedDataOutputStream(new FileOutputStream(toName)));
	}
	
	public void detectorStage() { scale(1.0 / instrumentGain); }
	
	public void readoutStage() { scale(instrumentGain); }
	
	public synchronized void clearData() {
		for(int t=0; t<frames; t++) {
			final LabocaFrame exposure = frame[t];
			if(exposure != null) for(int c=0; c<channels; c++) exposure.data[c] = 0.0F;
		}
	}
	
	public synchronized void randomData() {
		Random random = new Random();
		
		float[] rms = new float[channels];
		for(int c=0; c<channels; c++) rms[c] = (float)(Math.sqrt(1.0/channel[c].weight));
		
		for(int t=0; t<frames; t++) {
			final LabocaFrame exposure = frame[t];
			if(exposure != null) for(int c=0; c<channels; c++) if(channel[c].flag == 0)
				exposure.data[c] = rms[c] * (float)random.nextGaussian();						
		}	
	}
	
	// Adds correlated noise with 1/f^2 spectrum
	public void addCorrelated(int[] channelSelection, float[] gain, double nyquistLevel) {		
		double prior = 0.0;
		Random random = new Random();
		for(int t=0; t<frames; t++) {
			double signal = prior + nyquistLevel * random.nextGaussian();
			prior = signal;
			LabocaFrame exposure = frame[t];
			if(exposure != null) for(int k=0; k<channelSelection.length; k++) {
				final LabocaPixel pixel = channel[channelSelection[k]];
				if(pixel.flag == 0) exposure.data[pixel.index] += gain[k] * signal;
			}
		}	
	}
	

	// Peak is specified in volts at the detector...
	public synchronized void addSources(Vector<GaussianSource> sources) {
		getSourceGains();
		
		double[] A = new double[sources.size()];
		Vector2D[] centerOffset = new Vector2D[sources.size()];
			
		for(int i=0; i<sources.size(); i++) {
			GaussianSource source = sources.get(i);
			double sigma = source.FWHM / 2.35;
			A[i] = -0.5 / (sigma * sigma);
			centerOffset[i] = new Vector2D(); 
			map.projection.project(source.coords, centerOffset[i]);
			
			System.err.print("  Source #" + (i+1) + " is at " + Util.f2.format(centerOffset[i].x/Unit.arcsec) + "," + Util.f2.format(centerOffset[i].y/Unit.arcsec));
			System.err.println(" peak = " + (source.peak / Unit.uV) + " uV");
		}
		
		for(int t=0; t < frames; t++) {
			final LabocaFrame exposure = frame[t];
			if(exposure != null) {
				final float[] residual = frame[t].data;
				final double frameGain = gain * frame[t].transmission; 

				for(int bol=0; bol < channels; bol++) if(channel[bol].flag == 0) {
					final double g = channel[bol].sourceGain * frameGain;
					final Vector2D offset = getMapOffsets(t, bol);
					
					for(int i=0; i<A.length; i++) {
						final GaussianSource source = sources.get(i);
						final double d = offset.distanceTo(centerOffset[i]);
						
						// Do not check for flags, to get a true difference image...
						residual[bol] += (float) (g * source.peak * Math.exp(A[i] * d * d));		
					}
				}	    
			}
		}
	}
	
	
	


	public void readDataPar(String fileName)  throws IOException, FitsException, HeaderCardException {
		Fits fits = new Fits(new File(fileName), fileName.endsWith(".gz"));

		BinaryTableHDU hdu = (BinaryTableHDU) fits.getHDU(1);
		readDataPar(hdu);
	}
	
	
	public void readDataPar(BinaryTableHDU hdu)  throws IOException, FitsException, HeaderCardException {
		Header header = hdu.getHeader();

		// Read in the full HDU data
		hdu.getData().getData();
		
		frames = header.getIntValue("NAXIS2");
		integrationTime = ((double[]) hdu.getRow(0)[hdu.findColumn("INTEGTIM")])[0];
		System.err.println("   Sampling at " + Util.f3.format(1.0/integrationTime) + " Hz.");

		int iMJD = hdu.findColumn("MJD");
		int iRA = hdu.findColumn("BASLONG");
		int iDEC = hdu.findColumn("BASLAT");
		int iAZ = hdu.findColumn("AZIMUTH");
		int iEL = hdu.findColumn("ELEVATIO");
		int iDX = hdu.findColumn("LONGOFF");
		int iDY = hdu.findColumn("LATOFF");
		int iChop = hdu.findColumn("WOBDISLN");
		int iPhase = hdu.findColumn("PHASE");

		System.err.println("   " + frames + " frames found (" + 
				Util.f1.format(frames * integrationTime / Unit.min) + " minutes).");


		frame = new LabocaFrame[frames];
		
		if(scramble) System.err.println("   !!! Scrambling position data (noise map only) !!!");
		
		for(int t=0, T=0; t<frame.length; t++) {
			LabocaFrame exposure = new LabocaFrame(this);
			Object[] row = hdu.getRow(T++);
			exposure.MJD = ((double[])row[iMJD])[0];
			exposure.RA = ((double[])row[iRA])[0] * Unit.deg;
			exposure.DEC = ((double[])row[iDEC])[0] * Unit.deg;
			exposure.AZ = ((double[])row[iAZ])[0] * Unit.deg;
			exposure.EL = ((double[])row[iEL])[0] * Unit.deg;
			exposure.dX = ((double[])row[iDX])[0] * Unit.deg;
			exposure.dY = ((double[])row[iDY])[0] * Unit.deg;
			if(iChop > 0) exposure.chopperPosition = ((double[])row[iChop])[0] * Unit.deg;
			exposure.chopperPhase = ((int[])row[iPhase])[0];	

			if(scramble) {
				exposure.dX *= -1.0;
				exposure.dY *= -1.0;
				exposure.RA = 2.0 * equatorial.RA() - exposure.RA;
				exposure.DEC = 2.0 * equatorial.DEC() - exposure.DEC;
			}
			
			exposure.zenithTau = zenithTau;
			
			//exposure.calc();
			
			frame[t] = exposure;
		}	


		String phase1 = header.getStringValue("PHASE1");	
		String phase2 = header.getStringValue("PHASE2");

		if(phase1 != null) {
			if(phase1.equalsIgnoreCase("WON")) onPhase = 1;
			else if(phase1.equalsIgnoreCase("WOFF")) offPhase = 1;
		}
		if(phase2 != null) {
			if(phase2.equalsIgnoreCase("WON")) onPhase = 2;
			if(phase2.equalsIgnoreCase("WOFF")) offPhase = 2;
		}

		if(isChopped) System.err.println("   " + (getNodPhase() > 0 ? "ON-OFF" : "OFF-ON"));

		horizontal = new HorizontalCoordinates(frame[frames/2].AZ, frame[frames/2].EL);
		PA = frame[frames/2].PA;
	}



	public void readMonitor(String fileName)  throws IOException, FitsException, HeaderCardException {
		if(!(useHe3Data || calcHe3Gains)) return;

		Fits fits = new Fits(new File(fileName), fileName.endsWith(".gz"));

		BinaryTableHDU hdu = (BinaryTableHDU) fits.getHDU(1);
		readMonitor(hdu);
	}
	
	public void readMonitor(BinaryTableHDU hdu)  throws IOException, FitsException, HeaderCardException {
		System.err.print("   Parsing/Interpolating He3 temperatures. ");
		Header header = hdu.getHeader();

		hdu.getData().getData();
		
		Vector<Vector2D> table = new Vector<Vector2D>();
		
		int iMJD = hdu.findColumn("MJD");
		int iLABEL = hdu.findColumn("MONPOINT");
		int iVALUE = hdu.findColumn("MONVALUE");
		
		int nFlagged = 0;

		int n = header.getIntValue("NAXIS2");
		for(int i=0; i<n; i++) {
			Object[] row = hdu.getRow(i);	
		
			if(((String) row[iLABEL]).equals("LABOCA_HE3TEMP")) {
				Vector2D entry = new Vector2D();
				entry.x = ((double[]) row[iMJD])[0];
				entry.y = ((double[]) row[iVALUE])[0];
				table.add(entry);
			}
		}

		int from = 0, to=0;

		// plus or minus in days;
		double smoothFWHM = he3TimeScale / Unit.day;
		double windowSize = 2.0 * smoothFWHM;

		double sigma = smoothFWHM / 2.35;

		for(int t=0; t<frames; t++) {
			double MJD0 = frame[t].MJD;

			// Adjust the smoothing window...
			while(MJD0 - table.get(from).x > windowSize || from == table.size()-1) from++;
			while(table.get(to).x - MJD0 < windowSize || to == table.size()-1) to++;

			double sum = 0.0, sumw = 0.0;

			for(int tt=from; tt<=to; tt++) {
				Vector2D entry = table.get(tt);
				double dev = (MJD0 - entry.x) / sigma;
				double w = Math.exp(-0.5*dev*dev);
				sum += w * entry.y;
				sumw += w;
			}

			frame[t].he3Temp = sum/sumw;
			if(Double.isNaN(frame[t].he3Temp)) {
				frame[t] = null;
				nFlagged++;
			}
		}


		System.err.println(nFlagged == 0 ? "All good :-)" : "Dropped " + nFlagged + " frames without reliable He3 data :-(");

	}


	public Vector2D[] getSmoothPosition(int n) {
		Vector2D[] pos = new Vector2D[frames];

		int empties = 0;
		double intX=0.0, intY=0.0;
		for(int t=0; t<n; t++) {
			if(frame[t] == null) empties++;
			else {	    
				intX += frame[t].dX;
				intY += frame[t].dY;
			}
		}

		int tot = frames-n/2-1;
		for(int t=n/2; t<tot; t++) {
			pos[t] = new Vector2D(intX/(n - empties), intY/ (n - empties));
			LabocaFrame oldFrame = frame[t-n/2];
			LabocaFrame newFrame = frame[t+n/2];

			if(newFrame == null) empties++;
			else {
				intX += newFrame.dX;
				intY += newFrame.dY;
			}

			if(oldFrame == null) empties--;
			else {
				intX -= oldFrame.dX;
				intY -= oldFrame.dY;
			}

		}

		pos[tot] = n > empties ? new Vector2D(intX/(n-empties), intY/(n-empties)) : null;

		return pos;
	}


	
	public double getAverageScanningVelocity(double smoothT) {
		int N = (int)Math.ceil(smoothT/integrationTime);
		Vector2D[] pos = getSmoothPosition(N);

		double[] v = new double[frames]; 
		int n = 0;
		
		int tot = frames-N/2-2;
		
		for(int t=N/2+1; t<tot; t++) {
			LabocaFrame exposure = frame[t];
			if(exposure != null) if((exposure.flag & LabocaFrame.SKIP_SOURCE) == 0) {
				double dX = (pos[t+1].x - pos[t-1].x);
				double dY = (pos[t+1].y - pos[t-1].y);
				v[n++] = Math.hypot(dX, dY) / (2.0 * integrationTime);
			}
		}
		
		return n > 0 ? ArrayUtil.median(v) : Double.NaN;
	}
	
	
	public int velocityCut(double minV, double maxV) { return velocityCut(minV, maxV, 0.5 * Unit.s); }
	
	
	public int velocityCut(double minV, double maxV, double smoothT) {
		Vector2D[] pos = getSmoothPosition((int)Math.ceil(smoothT/integrationTime));

		double minD = minV*2.0*integrationTime;	
		double maxD = maxV*2.0*integrationTime;	

		int cut = 2;

		for(int t=1; t<frames-1; t++) if(frame[t] != null) {
			if(pos[t+1] == null || pos[t-1] == null) {
				frame[t].flag |= LabocaFrame.SKIP_SOURCE;
				cut++;
			}
			else {
				double D = Math.hypot(pos[t+1].x - pos[t-1].x, pos[t+1].y - pos[t-1].y);
				
				if(D < minD || D > maxD) {
					frame[t].flag |= LabocaFrame.SKIP_SOURCE;
					cut++;
				}
			}
		}

		return cut;
	}

	public int accelerationCut(double maxA) {
		Vector2D[] pos = getSmoothPosition(10);

		double maxD2 = maxA*integrationTime*integrationTime;
		maxD2 *= maxD2;	

		int cut = 0;

		for(int t=1; t<frames-1; t++) if(frame[t] != null) {
			if(pos[t+1] == null || pos[t-1] == null) {
				frame[t] = null;
				cut++;
			}
			else {
				double dX = (pos[t+1].x + pos[t-1].x - 2.0*pos[t].x);
				double dY = (pos[t+1].y + pos[t-1].y - 2.0*pos[t].y);
				if((dX*dX+dY*dY) > maxD2) {
					frame[t] = null;
					cut++;
				}
			}
		}

		return cut;
	}

	public Vector2D[] getAccelerations() { return getAccelerations(0.5); }
	
	public Vector2D[] getAccelerations(double smoothT) {
		Vector2D[] pos = getSmoothPosition((int)Math.ceil(smoothT/integrationTime));
		Vector2D[] accel = new Vector2D[frames];

		for(int t=1; t<frames-1; t++) {
			if(pos[t+1] == null || pos[t-1] == null) accel[t] = null;
			else {
				accel[t] = new Vector2D();
				accel[t].x = (pos[t+1].x + pos[t-1].x - 2.0*pos[t].x);
				accel[t].y = (pos[t+1].y + pos[t-1].y - 2.0*pos[t].y);
			}
		}

		return accel;
	}

	public void pointingAt(Vector2D dAZdEL) {
		for(int c=0; c<channels; c++)
			channel[c].position.math('-', dAZdEL);
	}

	public synchronized void highPassFilter(double T) {
		comments += "H";
		
		int nFFT = ArrayUtil.getPaddedFFTSize(frames);
		int nf = nFFT/2+1;
		double[] signal = new double[nFFT];

		// sigmat sigmaw = 1 ->
		// Dt sigmaw = 2.35 
		// Dt 2Pi sigmaf = 2.35 --> sigmaf = 2.35/2Pi 1/Dt
		// df = 1/Nt;
		// sigmaf / df = 2.35/2Pi * Nt/Dt 
	
		double sigma_f = 2.35/(2.0*Math.PI) * frames * integrationTime / T;		
		double A = 0.5/(sigma_f*sigma_f);

		Complex[] spectrum = new Complex[nf];
		for(int f=0; f<nf; f++) spectrum[f] = new Complex();

		for(int bol=0; bol<channels; bol++) {
			for(int t=0; t<frames; t++) {
				LabocaFrame exposure = frame[t];
				if(exposure == null) signal[t] = 0.0;
				else if((exposure.flag & LabocaFrame.MODELING_FLAGS) != 0) signal[t] = 0.0;
				else if(exposure.sampleFlag[bol] != 0) signal[t] = 0.0;
				else signal[t] = frame[t].data[bol];
			}
			Arrays.fill(signal, frames, signal.length, 0.0);
			ArrayUtil.uncheckedFFT(signal, spectrum);

			//for(int f=0; f<spectrum.length; f++) spectrum[f].scale(Math.exp(-A*f*f));
			int fmin = Math.min(spectrum.length, (int) Math.ceil(2.35*sigma_f));
			for(int f=fmin; f<spectrum.length; f++) spectrum[f].scale(0.0);

			ArrayUtil.uncheckedBackFFT(spectrum, signal);

			for(int t=0; t<frames; t++) if(frame[t] != null) frame[t].data[bol] -= (float) signal[t];
		}
	}	
	
	public synchronized void downsample(int n) {
		System.err.print(" Downsampling by " + n);

		int windowSize = (int)Math.round(1.82*ArrayUtil.getEquivalentWindowWidth("Hann")*n);
		double[] w = ArrayUtil.getWindowFunction("Hann", windowSize);

		LabocaFrame[] newFrame = new LabocaFrame[(frame.length-windowSize)/n+1]; 
		double[] newValue = new double[channels];

		System.err.println(" to " + newFrame.length + " frames.");


		for(int T=0, fromt=0; fromt+windowSize <= frames; T++, fromt += n) {
			Arrays.fill(newValue, 0.0);
			boolean hasEmpty = false;
			for(int t=fromt, blockt=0; blockt<windowSize; t++, blockt++) 
				for(int c=0; c<channels; c++) {
					if(frame[t] == null) { hasEmpty = true; break; }
					else newValue[c] += w[blockt] * frame[t].data[c];
				}
			newFrame[T] = hasEmpty ? null : frame[fromt+windowSize/2];
			if(!hasEmpty) for(int c=0; c<channels; c++) newFrame[T].data[c] = (float) newValue[c];
		}

		integrationTime *= n;
		frame = newFrame;
		frames = frame.length;
	}

	public void checkForNaNs() {
		System.err.print("?");
		for(int t=0; t<frame.length; t++) for(int c=0; c<channel.length; c++) {
			/*
	    if(Float.isNaN(frame[t].data[c])) {
		System.err.println("NaN: " + t + "," + c);
		System.exit(1);

	    if(Float.isInfinite(frame[t].data[c])) {
		System.err.println("Inf: " + t + "," + c);
		System.exit(1);
	    }
			 */
			if(frame[t] != null) if(frame[t].data[c] > 10.0) {
				System.err.println("Big: " + t + "," + c);
				System.exit(1);
			}
		}

	}

	public synchronized void shift(double dt) {
		double[][] coords = new double[7][frames];
		for(int t=0; t<frames; t++) {
			if(frame[t] != null) {
				if((frame[t].flag & LabocaFrame.BAD_DATA) == 0) {
					coords[0][t] = frame[t].RA;
					coords[1][t] = frame[t].DEC;
					coords[2][t] = frame[t].LST;
					coords[3][t] = frame[t].AZ;
					coords[4][t] = frame[t].EL;
					coords[5][t] = frame[t].dX;
					coords[6][t] = frame[t].dY;
				}
				else for(int i=0; i<7; i++) coords[i][t] = Double.NaN;
			}
			else for(int i=0; i<7; i++) coords[i][t] = Double.NaN;
		}

		for(int i=0; i<7; i++) coords[i] = ArrayExtras.shift(coords[i], dt/integrationTime, 0.5);

		for(int t=0; t<frames; t++) if(frame[t] != null) {
			if(Double.isNaN(coords[0][t])) frame[t] = null;
			else {
				frame[t].RA = coords[0][t];
				frame[t].DEC = coords[1][t];
				frame[t].LST = coords[2][t];
				frame[t].AZ = coords[3][t];
				frame[t].EL = coords[4][t];
				frame[t].dX = coords[5][t];
				frame[t].dY = coords[6][t];		
			}
		}
	}


	public void printASCIITimeStream() throws IOException {
		String filename = MiniCRUSH.workPath + File.separator + serialNo + "-" + subscanNo + ".tms";
		PrintStream out = new PrintStream(new BufferedOutputStream(new FileOutputStream(filename), 1000000));
		out.println(Util.e3.format(1.0/integrationTime));
		for(int t=0; t<frames; t++) {
			boolean isEmpty = true;
			if(frame[t] != null) if((frame[t].flag & LabocaFrame.BAD_DATA) == 0) {
				isEmpty = false;
				for(int c=0; c<channels; c++) 
					out.print(Util.e5.format((frame[t].sampleFlag[c] & LabocaFrame.SAMPLE_SPIKE_FLAGS) != 0 ? 0.0 : frame[t].data[c]) + "\t");
			}
			if(isEmpty) for(int c=0; c<channels; c++) out.print(Util.e5.format(0.0) + "\t");
			out.println();
				
		}
		out.flush();
		out.close();
		System.err.println("Written ASCII time-streams to " + filename);
	}
	
	public double[][] getCovariance() {
		System.err.print(" Calculating Covariance Matrix (this may take a while...)");
		
		double[][] covar = new double[channels][channels];
		int[][] n = new int[channels][channels];
		
		for(int t=0; t<frames; t++) if(frame[t] != null) if((frame[t].flag & LabocaFrame.SOURCE_FLAGS) == 0) {
			LabocaFrame exposure = frame[t];
			float[] data = exposure.data;

			for(int c1=0; c1<channels; c1++) if(channel[c1].flag == 0) if(exposure.sampleFlag[c1] == 0)
				for(int c2=c1+1; c2<channels; c2++) if(channel[c2].flag == 0) if(exposure.sampleFlag[c2] == 0) {
					covar[c1][c2] += exposure.relativeWeight * data[c1] * data[c2];
					n[c1][c2]++;
				}
		}
			
		for(int c1=0; c1<channels; c1++) for(int c2=c1+1; c2<channels; c2++) {
			covar[c1][c2] *= Math.sqrt(channel[c1].weight * channel[c2].weight) / n[c1][c2];
			covar[c2][c1] = covar[c1][c2];
		}

		System.err.println();
		
		return covar;
	}
    	
	public static double[][] getGroupCovariance(int[][] channelSelection, double[][] covar) {
		if(channelSelection == null) return null;

		int n = 0;
		for(int g=0; g<channelSelection.length; g++) n+= channelSelection[g].length;

		double[][] groupedCovar = new double[n][n];

		for(int k=0; k<n; k++) Arrays.fill(groupedCovar[k], Double.NaN);

		for(int g1=0, k1=0; g1<channelSelection.length; g1++) {
			for(int c1=0; c1<channelSelection[g1].length; c1++,k1++)
				for(int g2=0, k2=0; g2<channelSelection.length; g2++) { 
					for(int c2=0; c2<channelSelection[g2].length; c2++,k2++) {
						groupedCovar[k1][k2] = covar[channelSelection[g1][c1]][channelSelection[g2][c2]];
					}
				}
		}

		return groupedCovar;
	}
	
	public double[][] getFullCovariance(double[][] covar) {
		double[][] fullCovar = new double[320][320];
		
		for(int i=0; i<fullCovar.length; i++) Arrays.fill(fullCovar[i], Double.NaN);
		
		for(int c1=0; c1<channels; c1++) for(int c2=0; c2<channels; c2++)
			fullCovar[channel[c1].backendIndex][channel[c2].backendIndex] = covar[c1][c2];
			
		return fullCovar;
	}
	
	public void writeCovariance(String name, double[][] covar) throws IOException, FitsException {
		if(!name.endsWith(".fits")) name += "-" + serialNo + ".fits";
		if(covar == null) return;
		Fits fits = new Fits();
		BasicHDU hdu = Fits.makeHDU(covar);
		fits.addHDU(hdu);
		fits.write(new BufferedDataOutputStream(new FileOutputStream(name)));	
		System.err.println(" Written " + name);
	}
	
	
	float[] arrayCorrelated, arrayGain;
	float[][] cableCorr, cableGain;
	float[][] ampCorr, ampGain;
	float[][] boxCorr, boxGain;
	float[][] waferCorr, waferGain;
	float[][] regionCorr, regionGain;
	float[][] accelTerm, accelCoeff;
	float[] allXGain, allYGain, skyX, skyY;

	float[][] createCorrelatedStorage(String channelGroupSpec) {
		int[][] channelGroup = getChannelGroup(channelGroupSpec);
		if(channelGroup == null) return null;
		 
		float[][] groupCorr = new float[channelGroup.length][];
		if(channelGroup != null) {
			groupCorr = new float[channelGroup.length][];
				for(int g=0; g<channelGroup.length; g++) groupCorr[g] = new float[frames];
		}
		return groupCorr;
	}
	
	float[][] createGainStorage(String channelGroupSpec, String gainResolution) {
		/*
		int gainT = frames;
		if(gainResolution == null || gainResolution.length() == 0);
		else {
			try { 
				int T = (int) Math.ceil(Double.parseDouble(gainResolution) / integrationTime);
				if(T > 0) gainT = Math.min(frames, T);					
			}
			catch(NumberFormatException e) {}			
		}
		*/
		//gainSpans.put(channelGroupSpec, gainT);
		
		int[][] channelGroup = getChannelGroup(channelGroupSpec);
		if(channelGroup == null) return null;
		
		float[][] groupGain = new float[channelGroup.length][];
		for(int g=0; g<channelGroup.length; g++) {
			groupGain[g] = new float[channelGroup[g].length];	
			Arrays.fill(groupGain[g], 1.0F);
		}

		return groupGain;	
	}
	
	public void createStorage() {	
		cableCorr = createCorrelatedStorage("cables");
		cableGain = createGainStorage("cables", options.get("gain.cables"));

		ampCorr = createCorrelatedStorage("amps");
		ampGain = createGainStorage("amps", options.get("gain.amps"));

		boxCorr = createCorrelatedStorage("boxes");
		boxGain = createGainStorage("boxes", options.get("gain.boxes"));
		
		waferCorr = createCorrelatedStorage("wafers");
		waferGain = createGainStorage("wafers", options.get("gain.wafers"));
		
		regionCorr = createCorrelatedStorage("regions");
		regionGain = createGainStorage("regions", options.get("gain.regions"));
		
		accelTerm = new float[2][frames];
		accelCoeff = new float[2][allChannels.length];

		allXGain = new float[allChannels.length];
		allYGain = new float[allChannels.length];

		arrayCorrelated = new float[frames];
		arrayGain = new float[allChannels.length];
		
		skyX = new float[frames];
		skyY = new float[frames];
				
		Vector2D[] accel = getAccelerations();

		for(int t=0; t<frames; t++) {
			accelTerm[0][t] = accel[t] == null ? 0.0F : (float)Math.abs(accel[t].x);
			accelTerm[1][t] = accel[t] == null ? 0.0F : (float)Math.abs(accel[t].y);
		}

		Arrays.fill(accelCoeff[0], 1.0F);
		Arrays.fill(accelCoeff[1], 1.0F);

		Vector2D arraycenter = channel[referenceChannel].position;

		for(int s=0; s<allChannels.length; s++) {
			LabocaPixel pixel = channel[allChannels[s]];
			Vector2D position = pixel.position;
			allXGain[s] = (float)(position.x - arraycenter.x);
			allYGain[s] = (float)(position.y - arraycenter.y);
			arrayGain[s] = (float) pixel.gain;
		}
	}
	
	
	public void getArrayCorrelated(boolean flag, boolean solveGains, boolean robust, boolean gainRobust) {
		comments += robust ? "[C]" : "C";
		getCorrelated(allChannels, arrayGain, arrayCorrelated, robust);
		if(solveGains) {
			getGains(allChannels, arrayGain, arrayCorrelated, 0.1, 10.0, flag ? LabocaPixel.FLAG_GAIN : 0, false, LabocaPixel.GAINS_BIDIRECTIONAL, gainRobust);
			applyGains(allChannels, arrayGain);
			if(flag) {
				census();
				comments += goodChannelCount + " ";
			}
		}
	}
	

	
	public void getCorrs(int[][] channelGroup, float[][] gCorr, float[][] gGain, boolean robust) {		
		for(int g=0; g<channelGroup.length; g++) getCorrelated(channelGroup[g], gGain[g], gCorr[g], robust);		
	}
	
	

	
	public void getGains(int[][] channelGroup, float[][] gCorr, float[][] gGain, double loGain, double highGain, int flagPattern, boolean robust) {		
		for(int g=0; g<channelGroup.length; g++) getGains(channelGroup[g], gGain[g], gCorr[g], loGain, highGain, flagPattern, false, LabocaPixel.GAINS_BIDIRECTIONAL, robust);
		census();
		comments += goodChannelCount + " ";	
	}
	
	

	public void getDrifts(int nFrames, boolean robust) {
	
		int p = (int)Math.round(Math.log(nFrames) / Math.log(2.0));
		if(p < 1) p = 1;
		driftN = (int)Math.round(Math.pow(2.0, p));
	
		comments += (robust ? "[D" : "D") + driftN + (robust ? "]" : "");

		if(map != null) {
			double crossingtime = map.beamFWHM / aveScanSpeed;
			for(int c=0; c<channels; c++) channel[c].sourceFiltering *= 1.0 - crossingtime / (driftN * integrationTime); 
		}
			
		for(int t=0; t < frames;) {
			int tot = t+driftN;
			if(frames < tot+driftN) tot = frames;
			removeOffsets(allChannels, t, tot, robust);
			t = tot;
		}
	}
	
	public void resetFiltering() {
		// Make the reset channel filtering the highpass-filtering value...
		for(int c=0; c<channels; c++) channel[c].sourceFiltering = channel[c].sourceWhitening;
	}
	
	public void calcGroupFiltering(int[] channelGroup, float[] gain) {		
		int n = 0;
		double sumw = 0.0;
		for(int k=0; k<channelGroup.length; k++) {
			LabocaPixel pixel = channel[channelGroup[k]];
			if((pixel.flag == 0)) { 
				sumw += pixel.weight * gain[k] * gain[k]; 
				if(pixel.weight > 0.0) n++; 
			}		
		}
		
		if(n > 1) for(int k=0; k<channelGroup.length; k++) {
			LabocaPixel pixel = channel[channelGroup[k]];
			if((pixel.flag == 0)) pixel.sourceFiltering *= 1.0 - pixel.weight * gain[k] * gain[k] / sumw;			
		}
	}
	
	
	protected double averageFiltering;
	
	public void calcAverageFiltering() {	
		double sumw = 0.0, sumwG2 = 0.0;
		for(int c=0; c<channels; c++) if(channel[c].flag == 0) {
			double G = channel[c].sourceGain;
			double w = channel[c].weight * G * G;
			sumwG2 += w * channel[c].sourceFiltering * channel[c].sourceFiltering;
			sumw += w;			
		}
		if(sumw > 0.0) averageFiltering = Math.sqrt(sumwG2 / sumw);
		else averageFiltering = 0.0;
	}
	
	
	public void getTwisting(boolean robust) {
		float[][] twist = new float[cableChannels.length][frames];
		float[][] twistGain = new float[cableChannels.length][];
		
		for(int g=0; g<twistGain.length; g++) {
			twistGain[g] = new float[cableChannels[g].length];
			
			double avepin = 0.0, sumw = 0.0;
			for(int c=0; c<twistGain[g].length; c++) {
				double G = cableGain[g][c];
				LabocaPixel pixel = channel[cableChannels[g][c]];
				if(pixel.flag == 0) {
					avepin += pixel.weight * G * pixel.pin;
					sumw += pixel.weight * G;
				}
			}
			if(sumw > 0.0) avepin /= sumw;
			
			for(int c=0; c<twistGain[g].length; c++)
				twistGain[g][c] = cableGain[g][c] * (channel[cableChannels[g][c]].pin - (float)avepin);
		}
		
		comments += robust ? "[t]" : "t";
		for(int g=0; g<cableChannels.length; g++) getCorrelated(cableChannels[g], twistGain[g], twist[g], robust);
	}
	
	
	public void getBoxCorrelated(boolean solveGains, boolean robust, boolean gainRobust) {
		comments += robust ? "[B]" : "B";
		getCorrs(boxChannels, boxCorr, boxGain, robust);
		if(solveGains) {
			getGains(boxChannels, boxCorr, boxGain, 0.0, 10.0, LabocaPixel.FLAG_BOX, gainRobust);
			for(int g=0; g<boxChannels.length; g++) 
				for(int k=0; k<boxChannels[g].length; k++) 
					channel[boxChannels[g][k]].boxGain = boxGain[g][k];			
		}	
		
		/*
		System.err.println();
		for(int i=0; i<boxChannels.length; i++) {
			System.err.print("Box " + (i+1) + ": ");
			for(int k=0; k<boxChannels[i].length; k++) {
				LabocaPixel pixel = channel[boxChannels[i][k]];
				System.err.print((pixel.backendIndex+1) + " ");
			}
			System.err.println();
		}
		*/
	}
	
	public void getCableCorrelated(boolean solveGains, boolean robust, boolean gainRobust) {
		comments += robust ? "[c]" : "c";
		getCorrs(cableChannels, cableCorr, cableGain, robust);
		if(solveGains) {
			getGains(cableChannels, cableCorr, cableGain, 0.0, 10.0, LabocaPixel.FLAG_CABLE, gainRobust);
			for(int g=0; g<cableChannels.length; g++) 
				for(int k=0; k<cableChannels[g].length; k++) 
					channel[cableChannels[g][k]].cableGain = cableGain[g][k];
		}
		
		/*
		System.err.println();
		for(int i=0; i<cableChannels.length; i++) {
			System.err.print("Cable " + (i+1) + ": ");
			for(int k=0; k<cableChannels[i].length; k++) {
				LabocaPixel pixel = channel[cableChannels[i][k]];
				System.err.print((pixel.backendIndex+1) + " ");
			}
			System.err.println();
		}
		*/
		
	}

	
	public void getAmpCorrelated(boolean solveGains, boolean robust, boolean gainRobust) {
		comments += robust ? "[a]" : "a";
		getCorrs(ampChannels, ampCorr, ampGain, robust);
		if(solveGains) {
			getGains(ampChannels, ampCorr, ampGain, 0.0, 10.0, LabocaPixel.FLAG_AMP, gainRobust);	
			for(int g=0; g<ampChannels.length; g++) 
				for(int k=0; k<ampChannels[g].length; k++) 
					channel[ampChannels[g][k]].ampGain = ampGain[g][k];
		}
	}

	public void getWaferCorrelated(boolean solveGains, boolean robust, boolean gainRobust) {
		comments += robust ? "[Q]" : "Q";
		getCorrs(waferChannels, waferCorr, waferGain, robust);
		if(solveGains) {
			getGains(waferChannels, waferCorr, waferGain, 0.0, 10.0, LabocaPixel.FLAG_WAFER, gainRobust);
			for(int g=0; g<waferChannels.length; g++) 
				for(int k=0; k<waferChannels[g].length; k++) 
					channel[waferChannels[g][k]].waferGain = waferGain[g][k];
		}
	}
	
	public void getRegionCorrelated(boolean solveGains, boolean robust, boolean gainRobust) {
		comments += robust ? "[r]" : "r";
		getCorrs(regionChannels, regionCorr, regionGain, robust);
		if(solveGains) {
			getGains(regionChannels, regionCorr, regionGain, 0.0, 10.0, LabocaPixel.FLAG_REGION, gainRobust);
			for(int g=0; g<regionChannels.length; g++) 
				for(int k=0; k<regionChannels[g].length; k++) 
					channel[regionChannels[g][k]].regionGain = regionGain[g][k];
		}
	}
	
	
	
	// sum wg (x-x0) C / sum w g^2 (x-x0)^2 = 0
	// sum wgx = x0 sum wg
	// x0 = sum wgx / sum wg
	
	public void getSkyGradients(boolean robust) {
		float[] xGain = new float[allXGain.length];
		float[] yGain = new float[allYGain.length];

		Vector2D center = new Vector2D();
		double sumw = 0.0;
		
		for(int c=0; c<channels; c++) {
			LabocaPixel pixel = channel[c];
			if(pixel.flag == 0) {
				center.addMultipleOf(pixel.position, pixel.weight * pixel.gain);
				sumw += pixel.weight * pixel.gain;
			}
		}
		if(sumw > 0.0) center.scale(1.0 / sumw);
		
		for(int c=0; c<channels; c++) {
			LabocaPixel pixel = channel[c];
			xGain[c] = (float) (pixel.gain * (pixel.position.x - center.x));
			yGain[c] = (float) (pixel.gain * (pixel.position.y - center.y));
		}
		
		comments += robust ? "[S]" : "S";
		getCorrelated(allChannels, xGain, skyX, robust);
		getCorrelated(allChannels, yGain, skyY, robust);
	}
	
	public void getAccelCoeffs(boolean robust) {
		comments += robust ? "[a]" : "a";
		getGains(allChannels, accelCoeff[0], accelTerm[0], 0.1, 10.0, LabocaPixel.FLAG_XACCEL, false, LabocaPixel.GAINS_BIDIRECTIONAL, robust);
		getGains(allChannels, accelCoeff[1], accelTerm[1], 0.1, 10.0, LabocaPixel.FLAG_YACCEL, false, LabocaPixel.GAINS_BIDIRECTIONAL, robust);
		census();
	}
	
	
	public double correlatedFiltering = Double.NaN;
	public double skyFiltering = Double.NaN;
	public double boxesFiltering = Double.NaN;
	public double cablesFiltering = Double.NaN;
	public double twistingFiltering = Double.NaN;
	public double ampFiltering = Double.NaN;
	public double waferFiltering = Double.NaN;
	
	
	public synchronized void offset(double value) {
		for(int t=0; t<frames; t++) if(frame[t] != null) {
			for(int c=0; c<channels; c++) if(channel[c].flag == 0) frame[t].data[c] += value;			
		}
	}
	
	public void writeSpectra(String windowName, int windowSize) throws IOException {
		String fileName = MiniCRUSH.workPath + File.separator + serialNo + "-" + subscanNo + ".spec";
		
		double[] w = ArrayUtil.getWindowFunction(windowName, windowSize);
		
		System.err.println("  Calculating Power spectra.");
		double[] data = new double[frames];
		double[][] spectrum = new double[channels][];
		for(int c=0; c<channels; c++) {
			for(int t=0; t<frames; t++) {
				LabocaFrame exposure = frame[t];
				if(exposure == null) data[t] = 0.0;
				else if((exposure.flag & LabocaFrame.MODELING_FLAGS) != 0) data[t] = 0.0;
				else if(exposure.sampleFlag[c] != 0) data[t] = 0.0;
				else data[t] = frame[t].data[c];
			}
			spectrum[c] = ArrayUtil.averagePower(data, w);
		}

		 double df = 1.0 / (integrationTime * windowSize);	    

		 PrintStream out = new PrintStream(new BufferedOutputStream(new FileOutputStream(fileName), 1000000));
		
		 out.println("# f(Hz)\t P(ch=1)...");
		 
		 for(int f=1; f<spectrum[0].length; f++) {
			 out.print(Util.e3.format(f*df));
			 for(int c=0; c<channels; c++) out.print("\t" + Util.e3.format(spectrum[c][f]/df));
			 out.println();
		 }
		 
		 out.flush();
		 out.close();

		 System.err.println("Written Power spectra to " + fileName);
		    
	}
	
	static String path = "/home/pumukli/data/laboca";
	// filtering will apply to points below clipping and blanking levels when models are solved before source...
}

