//:::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::: /** @author John Miller * @version 1.2 * @date Sat Sep 8 13:53:16 EDT 2012 * @see LICENSE (MIT style license file). */ package scalation.analytics.classifier.par import java.util.concurrent.ForkJoinPool import scala.collection.parallel.ForkJoinTaskSupport import scala.math.min import scalation.linalgebra._ import scalation.linalgebra.gen.HMatrix3 import scalation.relalgebra.Relation import scalation.linalgebra.par.{MatrixD, VectorD} import scalation.util.time //::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::: /** The `NaiveBayes` class implements an Integer-Based Naive Bayes Classifier, * which is a commonly used such classifier for discrete input data. The * classifier is trained using a data matrix 'x' and a classification vector 'y'. * Each data vector in the matrix is classified into one of 'k' classes numbered * 0, ..., k-1. Prior probabilities are calculated based on the population of * each class in the training-set. Relative posterior probabilities are computed * by multiplying these by values computed using conditional probabilities. The * classifier is naive, because it assumes feature independence and therefore * simply multiplies the conditional probabilities. *----------------------------------------------------------------------------- * @param x the integer-valued data vectors stored as rows of a matrix * @param y the class vector, where y(l) = class for row l of the matrix x, x(l) * @param fn the names for all features/variables * @param k the number of classes * @param cn the names for all classes * @param vc the value count (number of distinct values) for each feature * @param me use m-estimates (me == 0 => regular MLE estimates) */ class NaiveBayes(x: MatriI, y: VectoI, fn: Array [String], k: Int, cn: Array [String], private var vc: VectoI = null, me: Int = 3) extends BayesClassifier (x, y, fn, k, cn) { private val DEBUG = false // debug flag private val cor = calcCorrelation // feature correlation matrix private var popC = new VectorI (k) // frequency counts for classes 0, ..., k-1 private val probC = new VectorD (k) // probabilities for classes 0, ..., k-1 private var popX = new HMatrix3 [Int] (k, n) // conditional frequency counts for variable/feature j private val probX = new HMatrix3 [Double] (k, n) // conditional probabilities for variable/feature j if (vc == null) vc = vc_fromData // set to default for binary data (2) popX.alloc (vc().toArray) probX.alloc (vc().toArray) if (DEBUG) { println ("value count vc = " + vc) println ("correlation matrix = " + cor) } // if //::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::: /** Build the model with feature order and selection * @param testStart starting index of test region (inclusive) used in cross-validation. * @param testEnd ending index of test region. (exclusive) used in cross-validation. */ def buildModel (testStart: Int, testEnd: Int): (Array[Boolean], DAG) = { (Array.fill (n)(true), new DAG(Array.ofDim [Int] (n, 0))) } // buildModel //::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::: /** Count the frequencies for 'y' having class 'i' and value 'x' for cases 0, 1, ... * Only the test region from 'testStart' to 'testEnd' is skipped, the rest is * training data. * @param testStart starting index of test region (inclusive) used in cross-validation * @param testEnd ending index of test region (exclusive) used in cross-validation */ def frequencies (testStart: Int, testEnd: Int) { val endworkers = 3 val size = (m/endworkers)+1 val popCw = Array.ofDim [VectorI] (endworkers) val popXw = Array.ofDim [HMatrix3[Int]] (endworkers) for(w <- 0 until endworkers){ popCw(w) = new VectorI (k) popXw(w) = new HMatrix3 [Int] (k, n) popXw(w).alloc(vc().toArray) } // for var temprange = (0 until endworkers).par temprange.tasksupport = new ForkJoinTaskSupport (new ForkJoinPool (endworkers)) for(w <- temprange) { for (l <- w * (size) until min((w+1)*size,m) if l < testStart || l >= testEnd) { // l = lth row of data matrix x val i = y(l) // get the class popCw (w)(i) += 1 // increment ith class for (j <- 0 until n) popXw (w)(i, j, x(l, j)) += 1 // increment ith class, jth feature, x value } // for } // for for (w <- 0 until endworkers){ popC += popCw (w) popX += popXw (w) } // for if (DEBUG) { println ("popC = " + popC) // #(C = i) println ("popX = " + popX) // #(X_j = x & C = i) } // if } // frequencies //::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::: /** Train the classifier by computing the probabilities for C, and the * conditional probabilities for X_j. * @param testStart starting index of test region (inclusive) used in cross-validation. * @param testEnd ending index of test region (exclusive) used in cross-validation. */ def train (testStart: Int, testEnd: Int) { frequencies (testStart, testEnd) // compute frequencies skipping test region val pcia = Array.ofDim [Double] (k) for (i <- 0 until k) { // for each class i pcia (i) = popC (i).toDouble // population of class i probC (i) = pcia (i) / md // probability of class i } // for for (j <- (0 until n).par) { // for each feature j for (i <- 0 until k) { val me_vc = me / vc (j).toDouble for (xj <- 0 until vc (j)) { // for each value for feature j: xj probX(i, j, xj) = (popX (i, j, xj) + me_vc) / (pcia (i) + me) } // for } // for } // for if (DEBUG) { println ("probC = " + probC) // P(C = i) println ("probX = " + probX) // P(X_j = x | C = i) } // if } // train //::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::: /** Given a discrete data vector 'z', classify it returning the class number * (0, ..., k-1) with the highest relative posterior probability. * Return the best class, its name and its relative probability. * @param z the data vector to classify */ def classify (z: VectoI): (Int, String, Double) = { val prob = new VectorD (k) for (i <- 0 until k) { prob(i) = probC (i) // P(C = i) for (j <- 0 until n) prob (i) *= probX (i, j, z(j)) // P(X_j = z_j | C = i) } // for if (DEBUG) println ("prob = " + prob) val best = prob.argmax () // class with the highest relative posterior probability (best, cn(best), prob(best)) // return the best class, its name and its probaility } // classify //::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::: /** Reset or re-initialize all the population and probability vectors and * hypermatrices to 0. */ def reset () { popC.set (0) probC.set (0) popX.set (0) probX.set (0) } // reset } // NaiveBayes class //:::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::: /** The `NaiveBayes` object is the companion object for the `NaiveBayes` class. */ object NaiveBayes { //::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::: /** Create a 'NaiveBayes' object, passing 'x' and 'y' together in one table. * @param xy the data vectors along with their classifications stored as rows of a matrix * @param fn the names of the features * @param k the number of classes * @param vc the value count (number of distinct values) for each feature * @param me use m-estimates (me == 0 => regular MLE estimates) */ def apply (xy: MatriI, fn: Array [String], k: Int, cn: Array [String], vc: VectoI = null, me: Int = 3) = { new NaiveBayes (xy(0 until xy.dim1, 0 until xy.dim2-1), xy.col(xy.dim2-1), fn, k, cn, vc, me) } // apply } // NaiveBayes object //::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::: /** The `NaiveBayesTest` object is used to test the `NaiveBayes` class. * Classify whether a car is more likely to be stolen (1) or not (1). * @see www.inf.u-szeged.hu/~ormandi/ai2/06-naiveBayes-example.pdf * > run-main scalation.analytics.classifier.par.NaiveBayesTest */ object NaiveBayesTest extends App { // x0: Color: Red (1), Yellow (0) // x1: Type: SUV (1), Sports (0) // x2: Origin: Domestic (1), Imported (0) // features: x0 x1 x2 val x = new MatrixI ((10, 3), 1, 0, 1, // data matrix 1, 0, 1, 1, 0, 1, 0, 0, 1, 0, 0, 0, 0, 1, 0, 0, 1, 0, 0, 1, 1, 1, 1, 0, 1, 0, 0) val y = VectorI (1, 0, 1, 0, 1, 0, 1, 0, 0, 1) // classification vector: 0(No), 1(Yes)) val fn = Array ("Color", "Type", "Origin") // feature/variable names val cn = Array ("No", "Yes") // class names println ("x = " + x) println ("y = " + y) println ("---------------------------------------------------------------") val nb = new NaiveBayes (x, y, fn, 2, cn) // create the classifier // train the classifier --------------------------------------------------- nb.train () // test sample ------------------------------------------------------------ val z1 = VectorI (1, 0, 1) // existing data vector to classify val z2 = VectorI (1, 1, 1) // new data vector to classify println ("classify (" + z1 + ") = " + nb.classify (z1) + "\n") println ("classify (" + z2 + ") = " + nb.classify (z2) + "\n") // cross validate the classifier nb.crossValidate () } // NaiveBayesTest object //::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::: /** The `NaiveBayesTest2` object is used to test the `NaiveBayes` class. * Given whether a person is Fast and/or Strong, classify them as making C = 1 * or not making C = 0 the football team. * > run-main scalation.analytics.classifier.par.NaiveBayesTest2 */ object NaiveBayesTest2 extends App { // training-set ----------------------------------------------------------- // x0: Fast // x1: Strong // y: Classification (No/0, Yes/1) // features: x0 x1 y val xy = new MatrixI ((10, 3), 1, 1, 1, 1, 1, 1, 1, 0, 1, 1, 0, 1, 1, 0, 0, 0, 1, 0, 0, 1, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0) val fn = Array ("Fast", "Strong") // feature names val cn = Array ("No", "Yes") // class names println ("xy = " + xy) println ("---------------------------------------------------------------") val nb = NaiveBayes (xy, fn, 2, cn, null, 0) // create the classifier // train the classifier --------------------------------------------------- nb.train () // test sample ------------------------------------------------------------ val z = VectorI (1, 0) // new data vector to classify println ("classify (" + z + ") = " + nb.classify (z) + "\n") nb.crossValidate () // cross validate the classifier } // NaiveBayesTest2 object //::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::: /** The `NaiveBayesTest3` object is used to test the `NaiveBayes` class. * > run-main scalation.analytics.classifier.par.NaiveBayesTest3 */ object NaiveBayesTest3 extends App { val filename = BASE_DIR + "breast-cancer.arff" var data = Relation (filename, -1, null) val xy = data.toMatriI2 (null) val fn = data.colName.toArray val cn = Array ("0", "1") // class names val nb = NaiveBayes (xy, fn, 2, cn, null, 0) // create the classifier nb.train() nb.crossValidate() } // NaiveBayesTest3 object