//:::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::: /** @author John Miller * @version 1.6 * @date Sat Dec 21 12:53:44 EST 2019 * @see LICENSE (MIT style license file). * * @title Model: Neural Network with 4+ Layers and Transfer Learning */ package scalation.analytics import scala.runtime.ScalaRunTime.stringOf import scalation.linalgebra.{FunctionV_2V, MatriD, MatrixD, MatriI, VectoD, VectoI} import scalation.util.banner import ActivationFun._ import Initializer._ import Optimizer.hp import PredictorMat2.{rescaleX, rescaleY} //:::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::: /** The `NeuralNet_XLT` class supports multi-output, multi-layer (input, {hidden} and output) * Neural-Networks with Transfer Learning. A layer (first hidden by default) from a neural- * network model trained on a related dataset is transferred into that position in 'this' model. * Given several input vectors and output vectors (training data), fit the parameters 'b' * connecting the layers, so that for a new input vector 'v', the net can predict the output vector. * @param x the m-by-nx input matrix (training data consisting of m input vectors) * @param y the m output vector (training data consisting of m output integer values) * @param nz the number of nodes in each hidden layer, e.g., Array (9, 8) => 2 hidden of sizes 9 and 8 * @param fname_ the feature/variable names (if null, use x_j's) * @param hparam the hyper-parameters for the model/network * @param f the array of activation function families between every pair of layers * @param transfer the saved network parameters from a layer of a related neural network * @param itran the inverse transformation function returns responses to original scale */ class NeuralNet_XLT (x: MatriD, y: MatriD, nz: Array [Int], fname_ : Strings = null, hparam: HyperParameter = hp, f: Array [AFF] = Array (f_tanh, f_tanh, f_sigmoid), transfer: NetParam = null, itran: FunctionV_2V = null) extends NeuralNet_XL (x, y, nz, fname_, hparam, f, itran) { b = (for (l <- layers) yield if (l == 1 && transfer != null) transfer else new NetParam (weightMat (sizes(l), sizes(l+1)), // parameters weights & weightVec (sizes(l+1)))).toArray // biases per active layer println (s"Create a NeuralNet_XLT with $nx input, ${stringOf (nz)} hidden and $ny output nodes: df_m = $df_m") } // NeuralNet_XLT //:::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::: /** The `NeuralNet_XLT` companion object provides factory functions for buidling three-layer * (one hidden layer) neural network classifiers. Note, 'rescale' is defined in `ModelFactory` * in Model.scala. */ object NeuralNet_XLT extends ModelFactory { private val DEBUG = false // debug flag //:::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::: /** Create a `NeuralNet_XLT` for a combined data matrix. * @param xy the combined input and output matrix * @param nz the number of nodes in each hidden layer, e.g., Array (5, 10) means 2 hidden with sizes 5 and 10 * @param fname the feature/variable names * @param hparam the hyper-parameters * @param af the array of activation function families over all layers * @param transfer the saved network parameters from a layer of a related neural network */ def apply (xy: MatriD, nz: Array [Int], fname: Strings = null, hparam: HyperParameter = hp, af: Array [AFF] = Array (f_tanh, f_tanh, f_id), transfer: NetParam = null): NeuralNet_XL = { var itran: FunctionV_2V = null // inverse transform -> original scale val (x, y) = pullResponse (xy) // assumes the last column is the response val x_s = if (rescale) rescaleX (x, af(0)) else x val y_s = if (af.last.bounds != null) { val y_i = rescaleY (y, af.last); itran = y_i._2; y_i._1 } else y if (DEBUG) println (s" scaled: x = $x_s \n scaled y = $y_s") new NeuralNet_XLT (x_s, MatrixD (Seq (y_s)), nz, fname, hparam, af, transfer, itran) } // apply } // NeuralNet_XLT object //:::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::: /** The `NeuralNet_XLTTest` object trains a neural netowrk on the `ExampleAutoMPG` dataset. * This test case does not use transfer learning. * A Neural Network with 2 hidden layers is created. * > runMain scalation.analytics.NeuralNet_XLTTest */ object NeuralNet_XLTTest extends App { import ExampleAutoMPG._ banner ("NeuralNet_XLT - no transfer learning - ExampleAutoMPG") banner ("NeuralNet_XLT with scaled y values") // hp("eta") = 0.0014 // try several values - train0 hp("eta") = 0.02 // try several values - train val nz = Array (7, 5) // sizes for two hidden layers val nn = NeuralNet_XLT (xy, nz) // factory function automatically rescales // val nn = new NeuralNet_XLT (x, MatrixD (Seq (y))) // constructor does not automatically rescale nn.train ().eval () // fit the weights using training data (0, 1, 2) println (nn.report) // banner ("cross-validation") // nn.crossValidate () } // NeuralNet_XLTTest object //:::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::: /** The `NeuralNet_XLTTest2` object trains a neural netowrk on the `ExampleAutoMPG` dataset. * This test case uses transfer learning. * A Neural Network with 2 hidden layers is created with the first hidden layer being * transferred from a model trained on related data. * FIX: find a related dataset for 'nn0'. * > runMain scalation.analytics.NeuralNet_XLTTest2 */ object NeuralNet_XLTTest2 extends App { import ExampleAutoMPG._ banner ("NeuralNet_XLT - transfer learning - ExampleAutoMPG") banner ("NeuralNet_XLT - train using related data") hp("eta") = 0.02 // try several values - train val nz0 = Array (7, 5) // sizes for two hidden layers val nn0 = NeuralNet_XLT (xy, nz0) // factory function automatically rescales // val nn0 = new NeuralNet_XLT (x, MatrixD (Seq (y))) // constructor does not automatically rescale nn0.train ().eval () // fit the weights using training data (0, 1, 2) println (nn0.report) val b1 = nn0.getNetParam () // get the paramaters (weights and biases) from the first hidden layer banner ("NeuralNet_XLT with scaled y values") // hp("eta") = 0.0014 // try several values - train0 hp("eta") = 0.02 // try several values - train val nz = Array (7, 5) // sizes for two hidden layers val nn = NeuralNet_XLT (xy, nz, transfer = b1) // factory function automatically rescales, transfer b1 // val nn = new NeuralNet_XLT (x, MatrixD (Seq (y))) // constructor does not automatically rescale nn.train ().eval () // fit the weights using training data (0, 1, 2) println (nn.report) // banner ("cross-validation") // nn.crossValidate () } // NeuralNet_XLTTest2 object