/*
 * MultiLayerPerceptron.java
 *
 * This class is a simple implementation of a three layer (single hidden layer)
 * perceptron, (mostly) as shown in Chapter 20
 * of Artificial Intelligence: A Modern Approach, 2nd Edition.
 *
 * Created on May 24, 2006, 8:04 PM
 */
/**
 *
 * @author  Bryan Pardo
 */

//package pardo.learning;
import java.util.*;
import java.lang.*;
import java.io.*;

public class MultiLayerPerceptron implements Serializable {

    // weights between layers
    private double [][] weight_input2hidden;  
    private double [][] weight_hidden2output; 
   
    // numbers of elements in layers
    private int numberOfInputs;
    private int numberOfOutputs;
    private int numberOfHidden;
    
    /** Creates a new instance of MultiLayerPerceptron. The perceptron
     * is created to be fully connected between adjacent
     * layers. Connection weights are set randomly between -0.5 and
     * 0.5.
     * 
     * @param numberofIn is the number of distinct input values each
     * training example will have
     * @param numberofHid is the number of hidden nodes to have 
     * @param numberofOut is the number of output nodes to have
     */
    public MultiLayerPerceptron(int numberOfIn, 
				int numberOfHid, 
				int numberOfOut) {
        numberOfInputs = numberOfIn; // we add one for the bias node 
        numberOfHidden = numberOfHid;// we add one for the bias node
        numberOfOutputs = numberOfOut;
        weight_input2hidden = new double [numberOfInputs][numberOfHidden]; 
	// weights from input to hidden layer
        weight_hidden2output = new double[numberOfHidden][numberOfOutputs];  
	// weights from hidden nodes to output layer
         
        // set the weights from the input layer to the hidden layer
        for (int k = 0; k < numberOfInputs; k++)
            for (int j = 0; j < numberOfHidden; j++)
                weight_input2hidden[k][j] =  (- 0.5 + Math.random());
        // set the weights from the hidden layer to the output layer
        for (int j = 0; j < numberOfHidden; j++)
            for (int i = 0; i < numberOfOutputs; i++)
                weight_hidden2output[j][i] = ( -0.5 + Math.random());    
    }
    
    /** This is a constructor that loads in a serialized MultiLayerPerceptron 
     *  from a file whose name and path are specified in the parameter
     *
     *  @param fullFilePath  This needs to be the full path name to the file 
     *                  containing the serialized MultiLayerPerceptron object
     *
     *  @throws IllegalArgumentException if we cant open a MultiLayerPerceptron 
     *          file by that name or if the file does not contain
     *          a serialized MultiLayerPerceptron
     */
    public MultiLayerPerceptron(String fullFilePath){
        MultiLayerPerceptron mlp = null;
        try{
            ObjectInputStream ois = new ObjectInputStream(new FileInputStream(fullFilePath));
            mlp = (MultiLayerPerceptron)ois.readObject();
            ois.close();
        }
        catch (ClassNotFoundException cnfe){throw new IllegalArgumentException(fullFilePath + " does not contain a serialized MultiLayerPerceptron");}
        catch (FileNotFoundException fnfe){throw new IllegalArgumentException(fullFilePath + " not found");}
        catch (IOException e){throw new IllegalArgumentException(fullFilePath + " : IOexception");}
       
        // weights between layers
        this.weight_input2hidden = mlp.weight_input2hidden;
        this.weight_hidden2output = mlp.weight_hidden2output;
        // numbers of elements in layers
        this.numberOfInputs = mlp.numberOfInputs;
        this.numberOfOutputs = mlp.numberOfOutputs;
        this.numberOfHidden = mlp.numberOfHidden;    
    }    
    
    /** this returns the sigmoid function from figure 20.16 (page 738)
     *  of Artificial Intelligence: A Modern Approach 2nd edition
     *
     * @param x  a double
     * @returns the value 1/(1+e^-x)
     */
    public double sigmoid(double x){
        return 1.0 /(1.0 + Math.exp(-x));

    }

    /** Do the back-prop-learning algorithm shown in figure 20.25
     *  (page 746) of Artificial Intelligence: A Modern Approach 2nd
     *  edition .
     *
     * Starting with the output layer this computes the desired change
     * for the output units, using the difference between the desired
     * output and the current output of the network. It then
     * propagates these changes back through previous layers. It does
     * this once in each epoch.  training continues until the desired
     * number of training epochs is completed
     *
     *@param inputAL a ArrayList of arrays of doubles. the ith array
     *in the ArrayList is the input in the ith training example
     *
     *@param desiredOutputAL a ArrayList of arrays of doubles. the ith
     *array in the ArrayList is the desired output of the ith training
     *example
     *
     *@param maxNumberOfEpochs says how many times the weights will be
     *updated in response
     *
     *@param learningRate is a value (should range from 0 to 1 ) that
     *speeds or slows learning to the training examples
     */
    public void backprop(ArrayList inputAL, 
			 ArrayList desiredOutputAL, 
			 int maxNumberOfEpochs, 
			 double learningRate){
        double input []  = new double[numberOfInputs];
        double hidden [] = new double[numberOfHidden];
        double output[]  = new double [numberOfOutputs];
        double desiredOutput [] = new double [numberOfOutputs];
        double deltaOutput [] = new double [numberOfOutputs];
        double deltaHidden [] = new double [numberOfHidden];
        
        // this loop runs once per epoch ( an epoch is a presentation
        // of all the training examples to the learner)
        for (int epoch = 1; epoch <= maxNumberOfEpochs; epoch++){
            
           // this loop runs once per training example
           for (int example = 0; example < inputAL.size(); example++){
               
                // load the input example
                input = (double [])inputAL.get(example);

                // load the desired output
                desiredOutput = (double [])desiredOutputAL.get(example);
               
                // calulate the values of the hidden  layer
                 for(int h = 0; h < numberOfHidden; h++){
                    for (int i = 0; i < numberOfInputs; i++)
                        hidden[h] = hidden[h]+input[i]*weight_input2hidden[i][h];
                    hidden[h] = sigmoid(hidden[h]);
                }

                // calulate the values of the output layer
                for(int u = 0; u < numberOfOutputs; u++){
                    for (int h = 0; h < numberOfHidden; h++)
                        output[u] = output[u]+hidden[h]*weight_hidden2output[h][u];
                    output[u] = sigmoid(output[u]);
                }  

               // for each output unit, calculate its error term
               for (int u = 0; u<output.length; u++){
                     deltaOutput[u] = output[u]*(1.0-output[u])*(desiredOutput[u]-output[u]);
               }
                               
               // calculate the desired change for each hidden node
               for (int h = 1; h<hidden.length; h++){
                    double totalBlame = 0;
                    for ( int u = 0; u < numberOfOutputs; u++){
                      totalBlame = totalBlame + weight_hidden2output[h][u] * deltaOutput[u];
                    }
                   deltaHidden[h] = hidden[h]*(1 - hidden[h]) * totalBlame;
               }
                
               //update the weights to the output layer
               for(int u = 0; u < numberOfOutputs; u++){
                    for (int h = 0; h < numberOfHidden; h++)
                        weight_hidden2output[h][u] = weight_hidden2output[h][u] + learningRate*deltaOutput[u]*hidden[h];
               }               
               
               //update the weights to the hidden layer (leaving out
               //the weights to the bias, which aren't used.
               for(int h = 0; h < numberOfHidden; h++){
                    for (int i = 0; i < numberOfInputs; i++)
                        weight_input2hidden[i][h] = weight_input2hidden[i][h] + learningRate*deltaHidden[h]*input[i];
               }                 
           }
        }
     }
    
    /** This function generates the output of the multilayer
     * perceptrion, given the input.
     * @param input an array of doubles (should be the same size as
     * the input layer of the perceptron)
     * @returns an array of doubles giving the output of the system
     */
    public double [] calcOutput(double [] input){
        double hidden [] = new double[numberOfHidden];
        double output[]  = new double [numberOfOutputs];
 
                // calulate the values of the hidden  layer
                 for(int h = 0; h < numberOfHidden; h++){
                    for (int i = 0; i < numberOfInputs; i++)
                        hidden[h] = hidden[h]+input[i]*weight_input2hidden[i][h];
                    hidden[h] = sigmoid(hidden[h]);
                }
               
                // calulate the values of the output layer
                for(int u = 0; u < numberOfOutputs; u++){
                    for (int h = 0; h < numberOfHidden; h++)
                        output[u] = output[u]+hidden[h]*weight_hidden2output[h][u];
                    output[u] = sigmoid(output[u]);
                }    
       return output;
}
    
    /** This function prints the values of the weights in the MLP to
     * the standard output */
    public void printWeights(){
        System.out.println("Weights:input (row) to hidden (column)");
        System.out.println("--------------------------------------");
        for (int i = 0; i < this.numberOfInputs; i++){
            System.out.print("input " + i +":");
            for (int h = 0; h < this.numberOfHidden; h++)
                System.out.print(" " + this.weight_input2hidden[i][h]);
            System.out.println("");
        }
            
        System.out.println("");
        System.out.println("Weights:hidden (row) to output (column)");
        System.out.println("--------------------------------------");
        for (int h = 0; h < this.numberOfHidden; h++){
            System.out.print("hidden " + h + ":");
            for (int u = 0; u < this.numberOfOutputs; u++)
                System.out.print(" " + this.weight_hidden2output[h][0]);
            System.out.println("");
        }
    }
    
    /** measure the error and output the average error per output
     * node...note that, since I round the values in the actual output
     * to 0 or 1, this means that I expect the desired outputs are
     * either 0s or 1s. In the case where the desired output is a
     * single boolean value...this measure returns the percentage of
     * correct classifications, given that we take the network's
     * single output node to be 1 when it exceeds .5 and 0
     * otherwise. */
    public  double  measureError(ArrayList inputAL, 
				 ArrayList desiredOutputAL){
        double totalError = 0;
        for (int i = 0;i < inputAL.size(); i++){
           double [] input = (double [])inputAL.get(i);
           double [] desired = (double [])desiredOutputAL.get(i);
           double [] actual = this.calcOutput(input); 
           for (int u = 0; u<desired.length;u++)
               totalError = totalError +  Math.abs(desired[u]- Math.round(actual[u]));
       }
       return totalError/(this.numberOfOutputs * inputAL.size());
    }
    
    /** 
     * Reads an input file to create testing and training examples
     *
     *  The input file is expected to be an ascii file, where the 1st line is ignored ");
     *   and lines two through n each consists of a fixed number of space-delimited 
     *   values. Each value must be 'true' or 'false'. What follows is an example 3-line 
     *   input file.
     *          
     *   GoodGrades GoodLetters GoodSAT IsRich HasScholarship ParentAlum SchoolActivities
     *   true  true  true  true  true  true  true  true
     *   true  true  true  true  true  true  false true
     *   
     *   Each line it taken as a single case. The first 'inputLayerSize' columns are taken
     *   to be the input. The remaining columns are taken as the desired output. If the 
     *   number of columns on any particular line does not agree with the sum of 
     *   'inputLayerSize' + 'outputLayerSize' the user is notified.
     * 
     * @param inputLayerSize 
     * @param outputLayerSize
     * @param filename  - a string containing the name of the file to be processed
     * @returns an ArrayList of two ArrayLists. The first ArrayList is the input.
     * the second is the output.
     */
    public static ArrayList readInputFile(int inputLayerSize, int outputLayerSize, String filename, boolean verbose){
	//read in the input file line by line
	BufferedReader inputBr = null;
        ArrayList inputAL = new java.util.ArrayList();
        ArrayList desiredOutputAL = new java.util.ArrayList();
        ArrayList outAL = new java.util.ArrayList(2);
	try {
	    inputBr = new BufferedReader(new FileReader(filename));
	    String line = inputBr.readLine();
            System.out.println( "reading file: " + filename);
            if (verbose){
                System.out.println( "");
                System.out.println( "attribute names: " + line );
            }
	    //we don't care about the first line as it is the
	    //attribute names
	    while ((line = inputBr.readLine()) != null){
		double [] input = new double[inputLayerSize];
		double [] desiredOutput = new double[outputLayerSize];

                if (verbose) {System.out.println("reading line: " + line);}
		//we have a line, so we have an example
		String patternStr = "[\\s\\t]+";
		String[] parts = line.split(patternStr);
		
                if (parts.length != (outputLayerSize + 	inputLayerSize)){		
                    System.out.print("Input error: this line has " + parts.length + " values, but the neural net expects ");
                    System.out.println( inputLayerSize + " input values and " + outputLayerSize + " output values for each case in the file" );
                }

		for (int j = 0; j < inputLayerSize; j++) {  
		    String value = parts[j].trim();
		    if (value.equalsIgnoreCase("true")) {
			input[j] = 1;
		    } else if (value.equalsIgnoreCase("false")) {
			input[j] = 0;
		    } else {
			System.out.println("Incorrect value: " + value +   " in input file.  Values should be true or false.");
                    }
                }
		for (int j = inputLayerSize; j < parts.length; j++) {  
		    String value = parts[j].trim();
		    if (value.equals("true")) {
			desiredOutput[j - inputLayerSize] = 1;
		    } else if (value.equals("false")) {
			desiredOutput[j - inputLayerSize] = 0;
		    } else {
			System.out.println("Incorrect value: " + value +   " in input file.  Values should be true or false.");
                    }
                }  
                inputAL.add(input);
		desiredOutputAL.add(desiredOutput);
            }
	}
	catch (FileNotFoundException ex) { ex.printStackTrace();}
	catch (IOException ex){ex.printStackTrace();}
	finally {
	    try {
		if (inputBr!= null) {
		    inputBr.close();
		}
	    }
	    catch (IOException ex) {ex.printStackTrace();}
	}
        outAL.add(inputAL);
        outAL.add(desiredOutputAL);
        return outAL;
    }
 
    private static void PrintUsage() {        
	System.out.println("Usage: java MultiLayerPerceptron inputLayerSize hiddenLayerSize outputLayerSize learningRate numberOfEpochs inputFileName");
        System.out.println("Example: java MultiLayerPerceptron " +  "7 8 1 1 10000 IvyLeague.txt");
        System.out.println("");
        System.out.println("inputLayerSize  - an integer specifying the number of nodes in the input layer");
        System.out.println("hiddenLayerSize - an integer specifying the number of nodes in the hidden layer");
        System.out.println("outputLayerSize - an integer specifying the number of nodes in the output layer");
        System.out.println("learningRate    - a real number that adjusts the learning rate. Keep this between 1 and 0");
        System.out.println("numberOfEpochs  - an integer specifying how many times the training set will be presented to the network");
        System.out.println("inputFileName   - the fully specified path to the input file. Note that windows pathnames may require ");
        System.out.println("                  double backslash '\\\\' in place of single backslash.");
        System.out.println("");
        System.out.println("This program makes a multilayer perceptron with the specified numbers");
        System.out.println("of input, output and hidden nodes. It then trains  the perceptron on the ");
        System.out.println("contents of the input file, dividing the input file into testing and training ");
        System.out.println("sets by randomly putting each case in either the training or testing set, with");
        System.out.println("equal probability. The network is trained on one set and tested on the other. ");
        System.out.println("Note that the size of the test and train sets won't always be equal, since ");
        System.out.println("cases are randomly assigned");
        System.out.println("");
        System.out.println("");
        System.out.println("");
        System.out.println("The input file is expected to be an ascii file, where the 1st line is ignored ");
        System.out.println("and lines two through n each consists of a fixed number of space-delimited ");
        System.out.println("values. Each value must be 'true' or 'false'. What follows is an example 3-line ");
        System.out.println("input file.");
        System.out.println("");       
        System.out.println("GoodGrades GoodLetters GoodSAT IsRich HasScholarship ParentAlum SchoolActivities");
        System.out.println("true  true  true  true  true  true  true  true");
        System.out.println("true  true  true  true  true  true  false true");
        System.out.println("");
        System.out.println("Each line it taken as a single case. The first 'inputLayerSize' columns are taken");
        System.out.println("to be the input. The remaining columns are taken as the desired output. If the ");
        System.out.println("number of columns on any particular line does not agree with the sum of ");
        System.out.println("'inputLayerSize' + 'outputLayerSize' the user is notified.");
        System.out.println("");
        System.out.println("");
        System.out.println("The output is a report of the  performance of the MultiLayerPerceptron on");
        System.out.println("the data before training and the performance after training.");

    }	   
    /**
     * This is a simple example of how to call and use the multi-layer
     * perceptron. Here, it is learning either a majority-rule (over
     * half the inputs == 1) or an xor-rule function.
     *
     * @param args the command line arguments
     */ 
    public static void main(String[] args) {
        
	if (args.length != 6) {
            PrintUsage();
            return;
        }	
	// set the parameters for the network architecture 
	int inputLayerSize = 0;     // = 5;
	int hiddenLayerSize = 0;    // = 8;
	int outputLayerSize = 0;    // = 1;
	// set the parameters for learning
	double  learningRate = 0;   // = 1;
	int numberOfEpochs = 0;     // = 10000;
	String inputFileName = "";
	String rawInputString = "";
        for (int i = 0; i < args.length; i++) {
	    if (i == 0) {
               Integer rawInputInt = new Integer(args[i]);
                inputLayerSize = rawInputInt.intValue();
	    } else if (i == 1) {
                Integer rawInputInt = new Integer(args[i]);
                hiddenLayerSize = rawInputInt.intValue();
	    } else if (i == 2) {
                Integer rawInputInt = new Integer(args[i]);
                outputLayerSize = rawInputInt.intValue();
	    } else if (i == 3) {
                Double rawInputDouble = new java.lang.Double(args[i]);
                learningRate = rawInputDouble.doubleValue();
	    } else if (i == 4) {
                Integer rawInputInt = new Integer(args[i]);
                numberOfEpochs = rawInputInt.intValue();
	    } else if (i == 5) {
		inputFileName = args[i];
	    }
	}

        // read in the data file with example cases 
        boolean verbose = true;
        ArrayList dataAL = readInputFile(inputLayerSize, outputLayerSize, inputFileName, verbose);
	ArrayList inputAL = (ArrayList)dataAL.get(0);
	ArrayList desiredOutputAL = (ArrayList)dataAL.get(1);	
        
        // make training and testing data sets
        int numberOfCases = inputAL.size();
        int numberOfTrainingCases = 0;
        int numberOfTestingCases = 0;
        ArrayList trainInputAL  = new ArrayList();
        ArrayList trainOutputAL = new ArrayList();
        ArrayList testInputAL  = new ArrayList();
        ArrayList testOutputAL = new ArrayList();
        
        // randomly assign cases to testing and training sets with equal probability
        for (int i = 0; i < numberOfCases; i++ ){
            if (java.lang.Math.round(java.lang.Math.random()) == 1){
                trainInputAL.add(inputAL.get(i));
                trainOutputAL.add(desiredOutputAL.get(i));
                numberOfTrainingCases += 1;
            }
            else{
                testInputAL.add(inputAL.get(i));
                testOutputAL.add(desiredOutputAL.get(i));
                numberOfTestingCases += 1;                
            }         
        }
 
       // now make a perceptron
       MultiLayerPerceptron mlp = new MultiLayerPerceptron(inputLayerSize,  
							   hiddenLayerSize,  
							   outputLayerSize);
       
       
       // see how well it works before training 
       double testScore = 1 - mlp.measureError(testInputAL, testOutputAL);
       double trainScore = 1 - mlp.measureError(trainInputAL, trainOutputAL);
       
       // train the system  
       System.out.println("");
       System.out.println("network connection weights before training");
       mlp.printWeights();
       mlp.backprop(trainInputAL, trainOutputAL, numberOfEpochs, learningRate);
       System.out.println("");
       System.out.println("network connection weights after training");
       mlp.printWeights();
       

        // report the results how well it works after training  
  
       System.out.println("");       
       System.out.println("inputLayerSize = " + inputLayerSize);
       System.out.println("hiddenLayerSize = " + hiddenLayerSize);
       System.out.println("outputLayerSize = " + outputLayerSize);
       System.out.println("learningRate = " + learningRate);
       System.out.println("numberOfEpochs = " + numberOfEpochs);
       System.out.println("inputFileName = " + inputFileName);  
       System.out.println("");
       System.out.println("total number of cases in input file: " + numberOfCases);
       System.out.println("number of training cases: " + numberOfTrainingCases);
       System.out.println("number of testing cases: " + numberOfTestingCases);
       System.out.println("");          
       System.out.println("TEST  SET, before training, proportion of correctly classified  cases = " + testScore);
       System.out.println("TRAIN SET, before training, proportion of correctly classified  cases = " + trainScore);
       System.out.println("");
       testScore = 1 - mlp.measureError(testInputAL, testOutputAL);       
       trainScore = 1 - mlp.measureError(trainInputAL, trainOutputAL);
       System.out.println("TEST  SET, after training, proportion of correctly classified  cases = " + testScore);
       System.out.println("TRAIN SET, after training, proportion of correctly classified  cases = " + trainScore);
           
    }    
			      
}