//::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::
/** @author  John Miller
 *  @version 1.6
 *  @date    Fri Mar 16 15:13:38 EDT 2018
 *  @see     LICENSE (MIT style license file).
 *
 *  @title   Model: Neural Network with 2 Layers (input and output layers)
 *
 *  @see     hebb.mit.edu/courses/9.641/2002/lectures/lecture03.pdf
 */

package scalation.analytics

import scala.collection.mutable.Set

import scalation.linalgebra.{FunctionV_2V, MatriD, MatrixD, VectoD, VectorD}
import scalation.math.noDouble
import scalation.plot.PlotM
import scalation.stat.Statistic
import scalation.util.banner
import java.text.SimpleDateFormat
import java.util.Date

import ActivationFun._
import Fit._
import Initializer._
import MatrixTransform._
import Optimizer._                                    // Optimizer - configuration
//import Optimizer_SGD._                              // Stochastic Gradient Descent
import Optimizer_SGDM._                               // Stochastic Gradient Descent with Momentum
//import Optimizer_ADAM._                             // Adaptive Moment Estimation
import PredictorMat2._
import StoppingRule._

//::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::
/** The `NeuralNet_2L` class supports multi-output, 2-layer (input and output)
 *  Neural-Networks.  It can be used for both classification and prediction,
 *  depending on the activation functions used.  Given several input vectors and output
 *  vectors (training data), fit the weights/parameters 'b' connecting the layers,
 *  so that for a new input vector 'z', the net can predict the output value, i.e.,
 *  <p>
 *      yp_j = f (b dot z)
 *  <p>
 *  where 'f' is the activation function and the parameters 'b' gives the
 *  weights between input and output layers.
 *  No batching is used for this algorithm.
 *  Note, 'b0' is treated as the bias, so 'x0' must be 1.0.
 *  @param x       the m-by-nx input matrix (training data consisting of m input vectors)
 *  @param y       the m-by-ny output matrix (training data consisting of m output vectors)
 *  @param fname_  the feature/variable names (if null, use x_j's)
 *  @param hparam  the hyper-parameters for the model/network
 *  @param f       the activation function family for layers 1->2 (input to output)
 *  @param itran   the inverse transformation function returns responses to original scale
 */
class NeuralNet_2L (x: MatriD, y: MatriD,
                    fname_ : Strings = null, hparam: HyperParameter = Optimizer.hp,
                    f: AFF = f_sigmoid, val itran: FunctionV_2V = null)
      extends PredictorMat2 (x, y, fname_, hparam)                       // sets eta in parent class
{
    private val DEBUG     = false                                        // debug flag
    private val bSize     = hp ("bSize").toInt                           // batch size
    private val maxEpochs = hp ("maxEpochs").toInt                       // maximum number of training epochs/iterations

    private var b = new NetParam (weightMat (nx, ny))                    // parameters (weights, bias implicit) in to out

    println (s"Create a NeuralNet_2L with $nx input and $ny output nodes")

    //::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::
    /** Return the parameters 'b' (weight matrix 'b.w') (array of 1).
     */
    def parameters: NetParams = Array (b)

    //::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::
    /** Given training data 'x_' and 'y_', fit the parameters 'b'.
     *  Minimize the error in the prediction by adjusting the parameters 'b'.
     *  The error 'ee' is simply the difference between the target value 'y' and the
     *  predicted value 'yp'.  Minimize the dot product of error with itself using
     *  gradient-descent. specifically move in the opposite direction of the gradient.
     *  Iterate over several epochs.  It does not use batching nor a sufficient stopping rule.
     *  In practice, use the 'train' or 'train2' methods that use better optimizers.
     *  @param x_  the training/full data/input matrix
     *  @param y_  the training/full response/output matrix
     */
    def train0 (x_ : MatriD = x, y_ : MatriD = y): NeuralNet_2L =
    {
        println (s"train0: eta = $eta")
        var sse0 = Double.MaxValue                                      // hold prior value of sse

        for (epoch <- 1 to maxEpochs) {                                 // iterate over each epoch
//          val yp = f.fM (b * x_)                                      // Yp = f(XB)
            val yp = f.fM (x_ *: b)                                     // Yp = f(XB)
            ee     = yp - y_                                            // negative of error matrix
            val d  = f.dM (yp) ** ee                                    // delta matrix for y
            b     -= x_.t * d * eta                                     // update 'b' parameters

            val sse = sseF (y_, f.fM (b * x_))                          // recompute sum of squared errors
            if (DEBUG) println (s"train0: parameters for $epoch th epoch: b = $b, sse = $sse")
            if (sse > sse0) return this                                 // return when sse increases
            sse0 = sse                                                  // save prior sse
        } // for
        this
    } // train0

    //::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::
    /** Given training data 'x_' and 'y_', fit the parameters 'b'.
     *  Minimize the error in the prediction by adjusting the parameters 'b'.
     *  Iterate over several epochs, where each epoch divides the training set into
     *  'nbat' batches.  Each batch is used to update the weights.
     *  @param x_  the training/full data/input matrix
     *  @param y_  the training/full response/output matrix
     */
    def train (x_ : MatriD = x, y_ : MatriD = y): NeuralNet_2L =
    {
        val epochs = optimize2 (x_, y_, b, eta, bSize, maxEpochs, f)     // optimize parameters b
        println (s"ending epoch = $epochs")
        estat.tally (epochs._2)
        this
    } // train

    //::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::
    /** Given training data 'x_' and 'y_', fit the parameters 'b'.
     *  Minimize the error in the prediction by adjusting the parameters 'b'.
     *  Iterate over several epochs, where each epoch divides the training set into
     *  'nbat' batches.  Each batch is used to update the weights.
     *  This version preforms an interval search for the best 'eta' value.
     *  @param x_  the training/full data/input matrix
     *  @param y_  the training/full response/output matrix
     */
    override def train2 (x_ : MatriD = x, y_ : MatriD = y): NeuralNet_2L =
    {
        val etaI = (0.25 * eta, 4.0 * eta)                               // quarter to four times eta
        val epochs = optimize2I (x_, y_, b, etaI, bSize, maxEpochs, f)    // optimize parameters b
        println (s"ending epoch = $epochs")
        estat.tally (epochs._2)
        this
    } // train2

    //::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::
    /** Build a sub-model that is restricted to the given columns of the data matrix.
     *  @param x_cols  the columns that the new model is restricted to
     */
    def buildModel (x_cols: MatriD): NeuralNet_2L =
    {
        new NeuralNet_2L (x_cols, y, null, hparam, f, itran)
    } // buildModel

    //::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::
    /** Given a new input vector 'z', predict the output/response vector 'f(z)'.
     *  @param z  the new input vector
     */
    def predictV (z: VectoD): VectoD = f.fV (b dot z)

    //::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::
    /** Given an input matrix 'z', predict the output/response matrix 'f(z)'.
     *  @param z  the input matrix
     */
    def predictV (z: MatriD = x): MatriD = f.fM (b * z)

} // NeuralNet_2L class


//::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::
/** The `NeuralNet_2L` companion object provides factory functions for buidling two-layer
 *  neural nets.  Note, 'rescale' is defined in `ModelFactory` in Model.scala.
 */
object NeuralNet_2L extends ModelFactory
{
    private val DEBUG = false                                          // debug flag

    //::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::
    /** Create a `NeuralNet_2L` for a combined data matrix.
     *  @param xy      the combined input and output matrix
     *  @param fname   the feature/variable names
     *  @param hparam  the hyper-parameters
     *  @param f       the activation function family for layers 1->2 (input to output)
     */
    def apply (xy: MatriD, fname:
               Strings = null, hparam: HyperParameter = Optimizer.hp,
               f: AFF = f_sigmoid): NeuralNet_2L =
    {
        var itran: FunctionV_2V = null                                 // inverse transform -> original scale
        val (x, y) = pullResponse (xy)                                 // assumes the last column is the response

        val x_s = if (rescale) rescaleX (x, f)
                  else x
        val y_s = if (f.bounds != null) { val y_i = rescaleY (y, f); itran = y_i._2; y_i._1 }
                  else y

        if (DEBUG) println (s" scaled: x = $x_s \n scaled y = $y_s")
        new NeuralNet_2L (x_s, MatrixD (Seq (y_s)), fname, hparam, f, itran)
    } // apply

   //::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::
    /** Create a `NeuralNet_2L` for a data matrix and response vector
     *  @param x       the input/data matrix
     *  @param y       the output/response vector
     *  @param fname   the feature/variable names
     *  @param hparam  the hyper-parameters
     *  @param f       the activation function family for layers 1->2 (input to output)
     */
    def apply (x: MatriD, y: VectoD,
               fname: Strings, hparam: HyperParameter,
               f: AFF): NeuralNet_2L =
    {
        val hp2 = if (hparam == null) Optimizer.hp else hparam
        var itran: FunctionV_2V = null                                 // inverse transform -> original scale

        val x_s = if (rescale) rescaleX (x, f)
                  else x
        val y_s = if (f.bounds != null) { val y_i = rescaleY (y, f); itran = y_i._2; y_i._1 }
                  else y

        if (DEBUG) println (s" scaled: x = $x_s \n scaled y = $y_s")
        new NeuralNet_2L (x_s, MatrixD (Seq (y_s)), fname, hp2, f, itran)
    } // apply

} // NeuralNet_2L object


//::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::
/** The `NeuralNet_2LTest` object is used to test the `NeuralNet_2L` class.  For this
 *  test, training data is used to fit the weights before using them for prediction.
 *  @see www4.rgu.ac.uk/files/chapter3%20-%20bp.pdf
 *  > runMain scalation.analytics.NeuralNet_2LTest
 */
object NeuralNet_2LTest extends App
{
    val s = 0                                                      // random number stream to use
    val x = new MatrixD ((3, 3), 1.0, 0.35, 0.9,                   // training data - input matrix (m vectors)
                                 1.0, 0.20, 0.7,
                                 1.0, 0.40, 0.95)
    val y = new MatrixD ((3, 2), 0.5, 0.4,                         // training data - output matrix (m vectors)
                                 0.3, 0.3,
                                 0.6, 0.5)

    println ("input  matrix x = " + x)
    println ("output matrix y = " + y)

    val nn = new NeuralNet_2L (x, y)                               // create a NeuralNet_2L model

//  for (eta <- 0.5 to 10.0 by 0.5) {
    for (i <- 1 to 20) {
        val eta = i * 0.5
        banner (s"NeuralNet_2LTest: Fit the parameters using optimization with learning rate $eta")

        nn.reset (eta_ = eta)
        nn.train0 ().eval ()                                       // fit the weights using training data
        println (nn.report)

        val yp = nn.predictV (x)                                   // predicted output values
        println ("target output:    y   = " + y)
        println ("predicted output: yp  = " + yp)
        println ("yp = " + nn.predict (x(0)))                      // predicted output values for row 0
    } // for

    banner ("NeuralNet_2LTest: Compare with Linear Regression - first column of y")

    val y0  = y.col(0)                                             // use first column of matrix y
    val rg0 = new Regression (x, y0)                               // create a Regression model
    println (rg0.analyze ().report)

    val y0p = rg0.predict (x)                                      // predicted output value
    println ("target output:    y0  = " + y0)
    println ("predicted output: y0p = " + y0p)

    banner ("NeuralNet_2LTest: Compare with Linear Regression - second column of y")

    val y1 = y.col(1)                                              // use second column of matrix y
    val rg1 = new Regression (x, y1)                               // create a Regression model
    println (rg1.analyze ().report)

    val y1p = rg1.predict (x)                                      // predicted output value
    println ("target output:    y1  = " + y1)
    println ("predicted output: y1p = " + y1p)

} // NeuralNet_2LTest object


//::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::
/** The `NeuralNet_2LTest2` object trains a neural netowrk on the `ExampleBasketBall` dataset.
 *  > runMain scalation.analytics.NeuralNet_2LTest2
 */
object NeuralNet_2LTest2 extends App
{
    import ExampleBasketBall._
    banner ("NeuralNet_2L vs. Regression - ExampleBasketBall")

//  val f_ = f_id                                                  // try different activation functions
    val f_ = f_sigmoid                                             // try different activation functions
//  val f_ = f_lreLU                                               // try different activation functions

    println ("ox = " + ox)
    println ("y  = " + y)

    banner ("Regression")
    val rg = Regression (oxy)
    println (rg.analyze ().report)

    banner ("prediction")                                          // not currently rescaling
    val yq = rg.predict ()                                         // scaled predicted output values for all x
    println ("target output:    y  = " + y)
    println ("predicted output: yq = " + yq)
    println ("error:            e  = " + (y - yq))

    banner ("NeuralNet_2L with scaled y values")
//  hp("eta") = 0.01                                               // try several values for train0
    hp("eta") = 0.1                                                // try several values for train
    val nn  = NeuralNet_2L (oxy, f = f_)                           // factory function automatically rescales
//  val nn  = new NeuralNet_2L (ox, y, f = f_)                     // constructor does not automatically rescale

    nn.trainSwitch (2).eval ()                                     // fit the weights using training data: 0, 1, 2
    println (nn.report)

/*
    banner ("scaled prediction")
    val yp = nn.predict ().col (0)                                 // scaled predicted output values for all x
    println ("target output:    y  = " + y)
    println ("predicted output: yp = " + yp)
    println ("error:            e  = " + (y - yp))

    banner ("unscaled prediction")
//  val (ymu, ysig) = (y.mean, sqrt (y.variance))                  // should obtain from apply - see below
//  val ypu = denormalizeV ((ymu, ysig))(yp)                       // denormalize predicted output values for all x
    val ypu = nn.itran (yp)                                        // denormalize predicted output values for all x
    println ("target output:   y   = " + y)
    println ("unscaled output: ypu = " + ypu)
*/

} // NeuralNet_2LTest2 object


//::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::
/** The `NeuralNet_2LTest3` object trains a neural netowrk on the `ExampleAutoMPG` dataset.
 *  > runMain scalation.analytics.NeuralNet_2LTest3
 */
object NeuralNet_2LTest3 extends App
{
    import ExampleAutoMPG._
    banner ("NeuralNet_2L vs. Regression - ExampleAutoMPG")

//  val f_ = f_id                                                  // try different activation functions
    val f_ = f_sigmoid                                             // try different activation functions
//  val f_ = f_lreLU                                               // try different activation functions

    banner ("Regression")
    val rg = Regression (oxy)
    println (rg.analyze ().report)

/*
    banner ("prediction")                                          // not currently rescaling
    val yq = rg.predict ()                                         // scaled predicted output values for all x
    println ("target output:    y  = " + y)
    println ("predicted output: yq = " + yq)
    println ("error:            e  = " + (y - yq))
*/

    banner ("NeuralNet_2L with scaled y values")
    hp("eta") = 0.05                                               // try several values for train0
//  hp("eta") = 0.2                                                // try several values for train
    val nn = NeuralNet_2L (oxy, f = f_)                            // factory function automatically rescales
//  val nn = new NeuralNet_2L (ox, y, f = f_)                      // constructor does not automatically rescale

    nn.trainSwitch (0).eval ()                                     // fit the weights using training data: 0, 1, 2
    println (nn.report)

/*
    banner ("scaled prediction")
    val yp = nn.predict ().col (0)                                 // scaled predicted output values for all x
    println ("target output:    y  = " + y)
    println ("predicted output: yp = " + yp)
    println ("error:            e  = " + (y - yp))

    banner ("unscaled prediction")
//  val (ymu, ysig) = (y.mean, sqrt (y.variance))                  // should obtain from apply - see below
//  val ypu = denormalizeV ((ymu, ysig))(yp)                       // denormalize predicted output values for all x
    val ypu = nn.itran (yp)                                        // denormalize predicted output values for all x
    println ("target output:   y   = " + y)
    println ("unscaled output: ypu = " + ypu)
*/

} // NeuralNet_2LTest3 object


//::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::
/** The `NeuralNet_2LTest4` object trains a neural netowrk on the `ExampleAutoMPG` dataset.
 *  It test cross-validation.
 *  > runMain scalation.analytics.NeuralNet_2LTest4
 */
object NeuralNet_2LTest4 extends App
{
    import ExampleAutoMPG._
    banner ("NeuralNet_2L cross-validation - ExampleAutoMPG")

    banner ("NeuralNet_2L with scaled y values")
//  hp("eta") = 0.001                                               // try several values - train0
    hp("eta") = 0.03                                                // try several values - train

    val nn = NeuralNet_2L (oxy)                                     // factory function automatically rescales
//  val nn = new NeuralNet_2L (x, MatrixD (Seq (y)))                // constructor does not automatically rescale

    nn.trainSwitch (2).eval ()                                      // fit the weights using training data (0, 1, 2)
    println (nn.report)

    banner ("cross-validation")
    nn.crossValidate ()

} // NeuralNet_2LTest4 object


//::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::
/** The `NeuralNet_2LTest5` object trains a neural network on the `ExampleAutoMPG` dataset.
 *  This tests forward feature/variable selection.
 *  > runMain scalation.analytics.NeuralNet_2LTest5
 */
object NeuralNet_2LTest5 extends App
{
    import ExampleAutoMPG._
    val n = x.dim2                                                  // number of parameters/variables
    val rSq = new MatrixD (n - 1, 3)                                // hold: R^2, R^2 Bar,  R^2 cv
    banner ("NeuralNet_2L feature selection - ExampleAutoMPG")

    banner ("NeuralNet_2L with scaled y values")
    hp("eta") = 0.3
    val nn = NeuralNet_2L (xy)                                      // factory function automatically rescales
//  val nn = new NeuralNet_2L (x, y)                                // constructor does not automatically rescale
    val dateFormatter = new SimpleDateFormat("dd/MM/yyyy hh:mm aa")
    var submittedDateConvert = new Date()
    var submittedAt = dateFormatter.format(submittedDateConvert)
    println("AKASH")
    println(submittedAt)
    nn.train ().eval ()                                             // fit the weights using training data
    println (nn.report)                                             // parameters and quality of fit
    val ft = nn.fitA(0)                                             // quality of fit for first output

    banner ("Forward Selection Test")
    nn.forwardSelAll ()

} // NeuralNet_2LTest5 object


//::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::
/** The `NeuralNet_2LTest6` object trains a neural network on the `ExampleAutoMPG` dataset.
 *  This tests forward feature/variable selection with plotting of R^2.
 *  > runMain scalation.analytics.NeuralNet_2LTest6
 */
object NeuralNet_2LTest6 extends App
{
    import ExampleAutoMPG._
    val n = ox.dim2                                                // number of parameters/variables
    banner ("NeuralNet_2L feature selection - ExampleAutoMPG")

    val f_ = f_sigmoid                                             // try different activation functions
//  val f_ = f_tanh                                                // try different activation functions
//  val f_ = f_lreLU                                               // try different activation functions
//  val f_ = f_id                                                  // try different activation functions

    banner ("NeuralNet_2L with scaled y values")
    hp("eta") = 0.02                                               // learning rate hyoer-parameter (see Optimizer)
    val nn  = NeuralNet_2L (oxy, f = f_)                           // factory function automatically rescales
//  val nn  = new NeuralNet_2L (ox, y, f = f_)                     // constructor does not automatically rescale

    nn.train ().eval ()                                            // fit the weights using training data
    println (nn.report)                                            // report parameters and fit
    val ft  = nn.fitA(0)                                           // fit for first output variable

    banner ("Forward Selection Test")
    val (cols, rSq) = nn.forwardSelAll ()                          // R^2, R^2 bar, R^2 cv
    println (s"rSq = $rSq")
    val k = cols.size
    println (s"k = $k, n = $n")
    val t = VectorD.range (1, k)                                   // instance index
    new PlotM (t, rSq.t, Array ("R^2", "R^2 bar", "R^2 cv"),
               "R^2 vs n for NeuralNet_2L", lines = true)

} // NeuralNet_2LTest6 object