//::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::
/** @author  John Miller
 *  @version 1.6
 *  @date    Fri Mar 16 15:13:38 EDT 2018
 *  @see     LICENSE (MIT style license file).
 *
 *  @title   Model: Neural Network with 3 Layers (input, hidden and output layers)
 *  @see     hebb.mit.edu/courses/9.641/2002/lectures/lecture03.pdf
 */

package scalation.analytics

import scala.collection.mutable.Set

import scalation.linalgebra.{FunctionV_2V, MatriD, MatrixD, VectoD, VectorD}
import scalation.math.noDouble
import scalation.plot.PlotM
import scalation.stat.Statistic
import scalation.util.banner

import ActivationFun._
import Fit._
import Initializer._
import MatrixTransform._
import Optimizer._                                  // Optimizer - configuration
//import Optimizer_SGD._                            // Stochastic Gradient Descent
import Optimizer_SGDM._                             // Stochastic Gradient Descent with Momentum
import PredictorMat2._
import StoppingRule._

//::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::
/** The `NeuralNet_3L1` class supports multi-output, 3-layer (input, hidden and output)
 *  Neural-Networks.  It can be used for both classification and prediction,
 *  depending on the activation functions used.  Given several input vectors and output
 *  vectors (training data), fit the parameters 'a' and 'b' connecting the layers,
 *  so that for a new input vector 'v', the net can predict the output value, i.e.,
 *  <p>
 *      yp = f1 (b * f0 (a * v))
 *  <p>
 *  where 'f0' and 'f1' are the activation functions and the parameter 'a' and 'b'
 *  are the parameters between input-hidden and hidden-output layers.
 *  Unlike `NeuralNet_2L` which adds input 'x0 = 1' to account for the intercept/bias,
 *  `NeuralNet_3L1` explicitly adds bias.
 *  @param x       the m-by-nx input matrix (training data consisting of m input vectors)
 *  @param y       the m-by-ny output vector (training data consisting of m output values)
 *  @param nz_     the number of nodes in hidden layer (-1 => use default formula)
 *  @param fname_  the feature/variable names (if null, use x_j's)
 *  @param hparam  the hyper-parameters for the model/network
 *  @param f0      the activation function family for layers 1->2 (input to hidden)
 *  @param f1      the activation function family for layers 2->3 (hidden to output)
 *  @param itran   the inverse transformation function returns responses to original scale
 */
class NeuralNet_3L1 (x: MatriD, y: VectoD,
                     nz_ : Int = -1,
                     fname_ : Strings = null, hparam: HyperParameter = hp,
                     f0: AFF = f_tanh, f1: AFF = f_id,
                     itran: FunctionV_2V = null)
      extends NeuralNet_3L (x, y, nz_, fname_, hparam, f0, f1, itran)
{
    protected var fit = fitA(0)                                        // fit for response (only one)

    println (s"Recreate a NeuralNet_3L1 with $nx input, $nz hidden and 1 output nodes: df_m = $df_m")

} // NeuralNet_3L1 class


//::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::
/** The `NeuralNet_3L1` companion object provides factory functions for buidling three-layer
 *  (one hidden layer) neural nets.  Note, 'rescale' is defined in `ModelFactory` in Model.scala.
 */
object NeuralNet_3L1 extends ModelFactory
{
    private val DEBUG = false                                          // debug flag

    //::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::
    /** Create a `NeuralNet_3L1` for a combined data matrix.
     *  @param xy      the combined input/data and output/response matrix
     *  @param nz      the number of nodes in hidden layer
     *  @param fname   the feature/variable names
     *  @param hparam  the hyper-parameters
     *  @param f0      the activation function family for layers 1->2 (input to output)
     *  @param f1      the activation function family for layers 2->3 (hidden to output)
     */
    def apply (xy: MatriD, nz: Int = -1,
               fname: Strings = null, hparam: HyperParameter = hp,
               f0: AFF = f_tanh, f1: AFF = f_id): NeuralNet_3L1 =
    {
        var itran: FunctionV_2V = null                                 // inverse transform -> original scale
        val (x, y) = pullResponse (xy)                                 // assumes the last column is the response

        val x_s = if (rescale) rescaleX (x, f0)
                  else x
        val y_s = if (f1.bounds != null) { val y_i = rescaleY (y, f1); itran = y_i._2; y_i._1 }
                  else y

        if (DEBUG) println (s" scaled: x = $x_s \n scaled y = $y_s")
        new NeuralNet_3L1 (x_s, y_s, nz, fname, hparam, f0, f1, itran)
    } // apply

    //::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::
    /** Create a `NeuralNet_3L1` for a data matrix and response vector.
     *  @param x       the m-by-nx input/data matrix
     *  @param y       the output/response m-vector
     *  @param nz      the number of nodes in hidden layer
     *  @param fname   the feature/variable names
     *  @param hparam  the hyper-parameters
     *  @param f0      the activation function family for layers 1->2 (input to output)
     *  @param f1      the activation function family for layers 2->3 (hidden to output)
     */
    def apply (x: MatriD, y: VectoD, nz: Int,
               fname: Strings, hparam: HyperParameter,
               f0: AFF, f1: AFF): NeuralNet_3L1 =
    {
        val hp2 = if (hparam == null) Optimizer.hp else hparam
        var itran: FunctionV_2V = null                                 // inverse transform -> original scale

        val x_s = if (rescale) rescaleX (x, f0)
                  else x
        val y_s = if (f1.bounds != null) { val y_i = rescaleY (y, f1); itran = y_i._2; y_i._1 }
                  else y

        if (DEBUG) println (s" scaled: x = $x_s \n scaled y = $y_s")
        new NeuralNet_3L1 (x_s, y_s, nz, fname, hp2, f0, f1, itran)
    } // apply

} // NeuralNet_3L1 object


//::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::
/** The `NeuralNet_3L1Test` object is used to test the `NeuralNet_3L1` class.
 *  @see www4.rgu.ac.uk/files/chapter3%20-%20bp.pdf
 *  > runMain scalation.analytics.NeuralNet_3L1Test
 */
object NeuralNet_3L1Test extends App
{
    val s = 0                                                      // random number stream to use
    val x = new MatrixD ((3, 3), 1.0, 0.35, 0.9,                   // training data - input matrix (m vectors)
                                 1.0, 0.20, 0.7,
                                 1.0, 0.40, 0.95)
    val y = VectorD (0.5, 0.3, 0.6)                                // training data - output vecrtore (m values)

    println ("input  matrix x = " + x)
    println ("output matrix y = " + y)

    hp("bSize") = 1
    val nn = new NeuralNet_3L1 (x.sliceCol (1, 3), y, 3)            // create a NeuralNet_3L1

    for (i <- 1 to 20) {
        val eta = i * 0.5
        banner (s"NeuralNet_3L1Test: Fit the parameters a & b using optimization with learning rate $eta")

        nn.reset (eta_ = eta)
        nn.train ().eval ()                                         // fit the weights using training data
        println (nn.report)

//      yp = nn.predictV (x)                                        // predicted output values
//      println ("target output:    y   = " + y)
//      println ("predicted output: yp  = " + yp)
        println ("yp0 = " + nn.predict (x(0)))                      // predicted output values for row 0
    } // for

    banner ("NeuralNet_3L1Test: Compare with Linear Regression")

    val rg0 = new Regression (x, y)                                 // create a Regression model
    println (rg0.analyze ().report)

    val yp = rg0.predict (x)                                       // predicted output value
    println ("target output:    y  = " + y)
    println ("predicted output: yp = " + yp)

} // NeuralNet_3L1Test object


//::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::
/** The `NeuralNet_3L1Test2` object trains a neural netowrk on the `ExampleBasketBall` dataset.
 *  > runMain scalation.analytics.NeuralNet_3L1Test2
 */
object NeuralNet_3L1Test2 extends App
{
    import ExampleBasketBall._
    banner ("NeuralNet_3L1 vs. Regession - ExampleBasketBall")

    println ("x = " + x)
    println ("y = " + y)

    banner ("Regression")
    val rg = Regression (oxy)
    println (rg.analyze ().report)

    banner ("prediction")                                           // not currently rescaling
    val yq = rg.predict ()                                          // scaled predicted output values for all x
    println ("target output:    y  = " + y)
    println ("predicted output: yq = " + yq)
    println ("error:            e  = " + (y - yq))

    banner ("NeuralNet_3L1 with scaled y values")
//  hp("eta") = 0.016                                               // try several values - train0
    hp("eta") = 0.1                                                 // try several values - train

    val nn = NeuralNet_3L1 (xy)                                      // factory function automatically rescales
//  val nn = new NeuralNet_3L1 (x, MatrixD (Seq (y)))                // constructor does not automatically rescale

    nn.trainSwitch (2).eval ()                                      // fit the weights using training data
    println (nn.report)

    banner ("scaled prediction")
    val yp = nn.predictV ().col (0)                                 // scaled predicted output values for all x
    println ("target output:    y  = " + y)
    println ("predicted output: yp = " + yp)
    println ("error:            e  = " + (y - yp))

/*
    banner ("unscaled prediction")
//  val (ymu, ysig) = (y.mean, sqrt (y.variance))                   // should obtain from apply - see below
//  val ypu = denormalizeV ((ymu, ysig))(yp)                        // denormalize predicted output values for all x
    val ypu = nn.itran (yp)                                         // denormalize predicted output values for all x
    println ("target output:   y   = " + y)
    println ("unscaled output: ypu = " + ypu)
*/

} // NeuralNet_3L1Test2 object


//::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::
/** The `NeuralNet_3L1Test3` object trains a neural netowrk on the `ExampleAutoMPG` dataset.
 *  > runMain scalation.analytics.NeuralNet_3L1Test3
 */
object NeuralNet_3L1Test3 extends App
{
    import ExampleAutoMPG._
    banner ("NeuralNet_3L1 vs. Regession - ExampleAutoMPG")

    banner ("Regression")
    val rg = Regression (oxy)
    println (rg.analyze ().report)

/*
    banner ("prediction")                                           // not currently rescaling
    val yq = rg.predict ()                                          // scaled predicted output values for all x
    println ("target output:    y  = " + y)
    println ("predicted output: yq = " + yq)
    println ("error:            e  = " + (y - yq))
*/

    banner ("NeuralNet_3L1 with scaled y values")
//  hp("eta") = 0.0014                                              // try several values - train0
    hp("eta") = 0.1                                                 // try several values - train

    val nn = NeuralNet_3L1 (xy)                                      // factory function automatically rescales
//  val nn = new NeuralNet_3L1 (x, MatrixD (Seq (y)))                // constructor does not automatically rescale

    nn.trainSwitch (2).eval ()                                      // fit the weights using training data (0, 1, 2)
    println (nn.report)

/*
    banner ("scaled prediction")
    val yp = nn.predict ().col (0)                                  // scaled predicted output values for all x
    println ("target output:    y  = " + y)
    println ("predicted output: yp = " + yp)
    println ("error:            e  = " + (y - yp))

    banner ("unscaled prediction")
//  val (ymu, ysig) = (y.mean, sqrt (y.variance))                   // should obtain from apply - see below
//  val ypu = denormalizeV ((ymu, ysig))(yp)                        // denormalize predicted output values for all x
    val ypu = nn.itran (yp)                                         // denormalize predicted output values for all x
    println ("target output:   y   = " + y)
    println ("unscaled output: ypu = " + ypu)
*/

} // NeuralNet_3L1Test3 object


//::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::
/** The `NeuralNet_3L1Test4` object trains a neural netowrk on the `ExampleAutoMPG` dataset.
 *  It test cross-validation.
 *  > runMain scalation.analytics.NeuralNet_3L1Test4
 */
object NeuralNet_3L1Test4 extends App
{
    import ExampleAutoMPG._
    banner ("NeuralNet_3L1 cross-validation - ExampleAutoMPG")

    banner ("NeuralNet_3L1 with scaled y values")
//  hp("eta") = 0.0014                                              // try several values - train0
    hp("eta") = 0.025                                               // try several values - train - set from best of `NeuralNet_3L1Test3`

    val nn = NeuralNet_3L1 (xy)                                      // factory function automatically rescales
//  val nn = new NeuralNet_3L1 (x, MatrixD (Seq (y)))                // constructor does not automatically rescale

    nn.trainSwitch (1).eval ()                                      // fit the weights using training data (0, 1, 2)
    println (nn.report)

    banner ("cross-validation")
    val stats = nn.crossValidate ()
    showQofStatTable (stats)

} // NeuralNet_3L1Test4 object


//::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::
/** The `NeuralNet_3L1Test5` object trains a neural network on the `ExampleAutoMPG` dataset.
 *  This tests forward feature/variable selection.
 *  > runMain scalation.analytics.NeuralNet_3L1Test5
 */
object NeuralNet_3L1Test5 extends App
{
    import ExampleAutoMPG._
    val n = x.dim2                                                  // number of parameters/variables
    val rSq = new MatrixD (n - 1, 3)                                // hold: R^2, R^2 Bar,  R^2 cv
    banner ("NeuralNet_3L1 feature selection - ExampleAutoMPG")

    banner ("NeuralNet_3L1 with scaled y values")
    hp("eta") = 0.3
    val nn = NeuralNet_3L1 (xy)                                      // factory function automatically rescales
//  val nn = new NeuralNet_3L1 (x, y)                                // constructor does not automatically rescale

    nn.train ().eval ()                                             // fit the weights using training data
    println (nn.report)                                             // parameters and quality of fit
    val ft = nn.fitA(0)                                             // quality of fit for first output

    banner ("Forward Selection Test")
    nn.forwardSelAll ()

} // NeuralNet_3L1Test5 object


//::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::
/** The `NeuralNet_3L1Test6` object trains a neural network on the `ExampleAutoMPG` dataset.
 *  This tests forward feature/variable selection with plotting of R^2.
 *  > runMain scalation.analytics.NeuralNet_3L1Test6
 */
object NeuralNet_3L1Test6 extends App
{
    import ExampleAutoMPG._
    val n = ox.dim2                                                // number of parameters/variables
    banner ("NeuralNet_3L1 feature selection - ExampleAutoMPG")

    val f_ = (f_sigmoid, f_id)                                     // try different activation functions
//  val f_ = (f_tanh, f_id)                                        // try different activation functions
//  val f_ = (f_lreLU, f_id)                                       // try different activation functions

    banner ("NeuralNet_3L1 with scaled y values")
    hp("eta") = 0.02                                               // learning rate hyoer-parameter (see Optimizer)
    val nn  = NeuralNet_3L1 (oxy, f0 = f_._1, f1 = f_._2)           // factory function automatically rescales
//  val nn  = new NeuralNet_3L1 (ox, y, f0 = f_._1, f1= f_._2)      // constructor does not automatically rescale

    nn.train ().eval ()                                            // fit the weights using training data
    println (nn.report)                                            // report parameters and fit
    val ft  = nn.fitA(0)                                           // fit for first output variable

    banner ("Forward Selection Test")
    val (cols, rSq) = nn.forwardSelAll ()                          // R^2, R^2 bar, R^2 cv
    println (s"rSq = $rSq")
    val k = cols.size
    println (s"k = $k, n = $n")
    val t = VectorD.range (1, k)                                   // instance index
    new PlotM (t, rSq.t, Array ("R^2", "R^2 bar", "R^2 cv"),
               "R^2 vs n for NeuralNet_3L1", lines = true)

} // NeuralNet_3L1Test6 object