{
 "metadata": {
  "name": "",
  "signature": "sha256:b3f4da5b401fc3911a6fa14e446f660b141ede1e9c80ecd1b470edfb58fc648f"
 },
 "nbformat": 3,
 "nbformat_minor": 0,
 "worksheets": [
  {
   "cells": [
    {
     "cell_type": "markdown",
     "metadata": {},
     "source": [
      "CROSS VALIDATION\n",
      "----------------\n",
      "\n",
      "You'll need some of this code and information to calculate your accuracy rate on your classifiers.\n",
      "\n",
      "EXAMPLE\n",
      "\n",
      "Let's say we have 10-fold cross validation...\n",
      "\n",
      "1. Divide test set into 10 random subsets.\n",
      "2. 1 test set is tested using the classifier trained on the remaining 9.\n",
      "3. We then do test/train on all of the other sets and average the percentages. \n",
      "\n",
      "To achieve the first step (divide our training set into k disjoint subsets), use the function [Kfold](http://scikit-learn.org/stable/modules/generated/sklearn.cross_validation.KFold.html) in the scikit.learn cross_validation package:\n",
      "\n",
      "    K-Folds cross validation iterator.\n",
      "    Provides train/test indices to split data in train test sets. Split dataset into k consecutive folds (without shuffling).\n",
      "\n",
      " You can visit the scikit.learn documentation to look at all the other options."
     ]
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "import numpy as np\n",
      "from sklearn import cross_validation\n",
      "from sklearn.neighbors import KNeighborsClassifier\n",
      "from sklearn import preprocessing\n",
      "from essentia.standard import MonoLoader\n",
      "from essentia.standard import ZeroCrossingRate, CentralMoments, Spectrum, Windowing, Centroid, DistributionShape\n",
      "import urllib2\n",
      "import urllib\n",
      "\n",
      "def process_corpus(corpus_URL):\n",
      "    \"\"\"Read a list of files to process from the text file at corpusURL. Return a list of URLs\"\"\" \n",
      "    # Open and read each line\n",
      "    url_list_text_data = urllib2.urlopen(corpus_URL) # it's a file like object and works just like a file\n",
      "    for file_URL in url_list_text_data: # files are iterable\n",
      "        yield file_URL.rstrip()\n",
      "        \n",
      "def spectral_features(filelist):\n",
      "    \"\"\"\n",
      "    Given a list of files, retrieve them, analyse the first 100mS of each file and return\n",
      "    a feature table.\n",
      "    \"\"\"\n",
      "    number_of_files = len(filelist)\n",
      "    number_of_features = 5\n",
      "    features = np.zeros([number_of_files, number_of_features])\n",
      "    sample_rate = 44100\n",
      "\n",
      "    for file_index, url in enumerate(filelist):\n",
      "        print url\n",
      "        urllib.urlretrieve(url, filename='/tmp/localfile.wav')\n",
      "        audio = MonoLoader(filename = '/tmp/localfile.wav', sampleRate = sample_rate)()\n",
      "        zcr = ZeroCrossingRate()\n",
      "        hamming_window = Windowing(type = 'hamming') # we need to window the frame to avoid FFT artifacts.\n",
      "        spectrum = Spectrum()\n",
      "        central_moments = CentralMoments()\n",
      "        distributionshape = DistributionShape()\n",
      "        spectral_centroid = Centroid()\n",
      "\n",
      "        frame_size = int(round(0.100 * sample_rate))   # 100ms\n",
      "        # Only do the first frame for now.\n",
      "        # TODO we should generate values for the entire file, probably by averaging the features.\n",
      "        current_frame = audio[0 : frame_size]\n",
      "        features[file_index, 0] = zcr(current_frame)\n",
      "        spectral_magnitude = spectrum(hamming_window(current_frame))\n",
      "        centroid = spectral_centroid(spectral_magnitude)\n",
      "        spectral_moments = distributionshape(central_moments(spectral_magnitude))\n",
      "        features[file_index, 1] = centroid\n",
      "        features[file_index, 2:5] = spectral_moments\n",
      "    return features\n",
      "\n",
      "def crossValidateKNN(features, labels):\n",
      "    \"\"\"\n",
      "    This code is provided as a template for your cross-validation\n",
      "    computation. Pass into the variables \"features\", \"labels\" your own data. \n",
      "\n",
      "    As well, you can replace the code in the \"BUILD\" and \"EVALUATE\" sections\n",
      "    to be useful with other types of Classifiers.\n",
      "    \"\"\"\n",
      "    #\n",
      "    # CROSS VALIDATION \n",
      "    # The features array is arranged as rows of instances, columns of features in our training set.\n",
      "    numInstances, numFeatures = features.shape\n",
      "    numFolds = min(10, numInstances) # how many cross-validation folds do you want - (default=10)\n",
      "    # divide test set into 10 random subsets\n",
      "    indices = cross_validation.KFold(numInstances, n_folds = numFolds)\n",
      "\n",
      "    errors = np.empty(numFolds)\n",
      "    for foldIndex, (train_index, test_index) in enumerate(indices):\n",
      "        # SEGMENT DATA INTO FOLDS\n",
      "        print('Fold: %d' % foldIndex) \n",
      "        print(\"TRAIN: %s\" % train_index)\n",
      "        print(\"TEST: %s\" % test_index)\n",
      "    \n",
      "        # SCALE\n",
      "        scaler = preprocessing.MinMaxScaler(feature_range = (-1, 1))\n",
      "        training_features = scaler.fit_transform(features.take(train_index, 0))\n",
      "        # BUILD NEW MODEL - ADD YOUR MODEL BUILDING CODE HERE...\n",
      "        model = KNeighborsClassifier(n_neighbors = 3)\n",
      "        model.fit(training_features, labels.take(train_index, 0))\n",
      "        # RESCALE TEST DATA TO TRAINING SCALE SPACE\n",
      "        testing_features = scaler.transform(features.take(test_index, 0))\n",
      "        # EVALUATE WITH TEST DATA - ADD YOUR MODEL EVALUATION CODE HERE\n",
      "        model_output = model.predict(testing_features)\n",
      "        print(\"KNN prediction %s\" % model_output) # Debugging.\n",
      "        # CONVERT labels(test,:) LABELS TO SAME FORMAT TO COMPUTE ERROR \n",
      "        labels_test = labels.take(test_index, 0)\n",
      "        # COUNT ERRORS. matches is a boolean array, taking the mean does the right thing.\n",
      "        matches = model_output != labels_test\n",
      "        errors[foldIndex] = matches.mean()\n",
      "    print('cross validation error: %f' % errors.mean())\n",
      "    print('cross validation accuracy: %f' % (1.0 - errors.mean()))"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 6
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "snares_URL = \"https://ccrma.stanford.edu/workshops/mir2014/SnareCorpus.txt\"\n",
      "snare_file_list = [audio_file_URL for audio_file_URL in process_corpus(snares_URL)]\n",
      "kicks_URL = \"https://ccrma.stanford.edu/workshops/mir2014/KickCorpus.txt\"\n",
      "kick_file_list = [audio_file_URL for audio_file_URL in process_corpus(kicks_URL)]\n",
      "all_files = snare_file_list + kick_file_list\n",
      "\n",
      "drum_labels = np.empty(len(all_files), np.int32)\n",
      "num_of_snares = len(snare_file_list)\n",
      "drum_labels[0:num_of_snares] = 1 # set labels to the first sample type, e.g. snare\n",
      "drum_labels[num_of_snares: num_of_snares + len(kick_file_list)] = 2 # Set next set to the second sample type, e.g kick\n",
      "print drum_labels\n",
      "\n",
      "drum_features = spectral_features(all_files)\n",
      "print drum_features\n",
      "\n",
      "print crossValidateKNN(drum_features, drum_labels)"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "[1 1 1 1 1 1 1 1 1 1 2 2 2 2 2 2 2 2 2 2]\n",
        "https://ccrma.stanford.edu/workshops/mir2014/audio/drum%20samples/snares/SNARE_01_01.WAV\n",
        "https://ccrma.stanford.edu/workshops/mir2014/audio/drum%20samples/snares/SNARE_02_01.WAV"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "https://ccrma.stanford.edu/workshops/mir2014/audio/drum%20samples/snares/SNARE_04_01.WAV\n",
        "https://ccrma.stanford.edu/workshops/mir2014/audio/drum%20samples/snares/SNARE_05_01.WAV"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "https://ccrma.stanford.edu/workshops/mir2014/audio/drum%20samples/snares/SNARE_06_01.WAV\n",
        "https://ccrma.stanford.edu/workshops/mir2014/audio/drum%20samples/snares/SNARE_07_01.WAV"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "https://ccrma.stanford.edu/workshops/mir2014/audio/drum%20samples/snares/SNARE_08_01.WAV\n",
        "https://ccrma.stanford.edu/workshops/mir2014/audio/drum%20samples/snares/SNARE_09_01.WAV"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "https://ccrma.stanford.edu/workshops/mir2014/audio/drum%20samples/snares/SNARE_10_01.WAV\n",
        "https://ccrma.stanford.edu/workshops/mir2014/audio/drum%20samples/snares/snare_mono.wav"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "https://ccrma.stanford.edu/workshops/mir2014/audio/drum%20samples/kicks/Bass_Drum_01_V01.WAV\n",
        "https://ccrma.stanford.edu/workshops/mir2014/audio/drum%20samples/kicks/Bass_Drum_02_V01.WAV"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "https://ccrma.stanford.edu/workshops/mir2014/audio/drum%20samples/kicks/Bass_Drum_03_V01.WAV"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "https://ccrma.stanford.edu/workshops/mir2014/audio/drum%20samples/kicks/Bass_Drum_04_V01.WAV"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "https://ccrma.stanford.edu/workshops/mir2014/audio/drum%20samples/kicks/Bass_Drum_05_V01.WAV"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "https://ccrma.stanford.edu/workshops/mir2014/audio/drum%20samples/kicks/Bass_Drum_06_V01.WAV"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "https://ccrma.stanford.edu/workshops/mir2014/audio/drum%20samples/kicks/Bass_Drum_07_V01.WAV"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "https://ccrma.stanford.edu/workshops/mir2014/audio/drum%20samples/kicks/Bass_Drum_08_V01.WAV"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "https://ccrma.stanford.edu/workshops/mir2014/audio/drum%20samples/kicks/Bass_Drum_09_V01.WAV"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "https://ccrma.stanford.edu/workshops/mir2014/audio/drum%20samples/kicks/Bass_Drum_10_V01.WAV"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "[[  1.29251704e-01   2.20017299e-01   4.07947935e-02   8.77821267e-01\n",
        "   -8.77261162e-03]\n",
        " [  1.06349207e-01   2.09325567e-01   3.52719985e-02   7.70348966e-01\n",
        "   -3.22882414e-01]\n",
        " [  1.14965983e-01   1.98087305e-01   3.39526162e-02   8.53771806e-01\n",
        "   -1.91451073e-01]\n",
        " [  1.09977327e-01   1.89880371e-01   3.07796430e-02   9.61023569e-01\n",
        "    2.17356920e-01]\n",
        " [  1.37641728e-01   2.61134952e-01   4.25882451e-02   6.05063379e-01\n",
        "   -4.61072683e-01]\n",
        " [  1.12925172e-01   2.12166250e-01   3.77560928e-02   9.84243870e-01\n",
        "    2.55144835e-01]\n",
        " [  1.43990934e-01   2.32887402e-01   4.25517298e-02   8.88096571e-01\n",
        "   -2.55122185e-02]\n",
        " [  1.24036282e-01   2.34317511e-01   4.48079035e-02   8.41823637e-01\n",
        "   -9.77532864e-02]\n",
        " [  1.33560091e-01   2.40808740e-01   3.97951268e-02   7.26550639e-01\n",
        "   -2.08487749e-01]\n",
        " [  1.14058957e-01   2.19529420e-01   3.08201890e-02   9.37722743e-01\n",
        "    6.40640974e-01]\n",
        " [  9.97732393e-03   4.34924886e-02   1.27002802e-02   3.57807994e+00\n",
        "    1.42459507e+01]\n",
        " [  7.02947844e-03   2.78101377e-02   6.62755780e-03   4.85479450e+00\n",
        "    2.90792427e+01]\n",
        " [  5.66893443e-03   1.62475649e-02   2.56283046e-03   7.87885380e+00\n",
        "    8.96873932e+01]\n",
        " [  5.66893443e-03   3.63209695e-02   9.78391711e-03   3.97197366e+00\n",
        "    1.83651676e+01]\n",
        " [  8.61677993e-03   2.82980110e-02   8.20268691e-03   5.25167370e+00\n",
        "    3.25443840e+01]\n",
        " [  7.93650839e-03   2.40181480e-02   5.99115109e-03   5.35486269e+00\n",
        "    3.59657173e+01]\n",
        " [  7.70975044e-03   2.50173863e-02   6.07906608e-03   5.63076735e+00\n",
        "    3.99530830e+01]\n",
        " [  5.66893443e-03   3.24563682e-02   9.35256947e-03   4.67505932e+00\n",
        "    2.59624958e+01]\n",
        " [  8.84353742e-03   3.49452570e-02   8.03499669e-03   4.01454067e+00\n",
        "    2.00484810e+01]\n",
        " [  8.61677993e-03   2.26410441e-02   6.27955049e-03   5.77101851e+00\n",
        "    3.90028992e+01]]"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "Fold: 0\n",
        "TRAIN: [ 2  3  4  5  6  7  8  9 10 11 12 13 14 15 16 17 18 19]\n",
        "TEST: [0 1]\n",
        "KNN prediction [1 1]\n",
        "Fold: 1\n",
        "TRAIN: [ 0  1  4  5  6  7  8  9 10 11 12 13 14 15 16 17 18 19]\n",
        "TEST: [2 3]\n",
        "KNN prediction [1 1]\n",
        "Fold: 2\n",
        "TRAIN: [ 0  1  2  3  6  7  8  9 10 11 12 13 14 15 16 17 18 19]\n",
        "TEST: [4 5]\n",
        "KNN prediction [1 1]\n",
        "Fold: 3\n",
        "TRAIN: [ 0  1  2  3  4  5  8  9 10 11 12 13 14 15 16 17 18 19]\n",
        "TEST: [6 7]\n",
        "KNN prediction [1 1]\n",
        "Fold: 4\n",
        "TRAIN: [ 0  1  2  3  4  5  6  7 10 11 12 13 14 15 16 17 18 19]\n",
        "TEST: [8 9]\n",
        "KNN prediction [1 1]\n",
        "Fold: 5\n",
        "TRAIN: [ 0  1  2  3  4  5  6  7  8  9 12 13 14 15 16 17 18 19]\n",
        "TEST: [10 11]\n",
        "KNN prediction [2 2]\n",
        "Fold: 6\n",
        "TRAIN: [ 0  1  2  3  4  5  6  7  8  9 10 11 14 15 16 17 18 19]\n",
        "TEST: [12 13]\n",
        "KNN prediction [2 2]\n",
        "Fold: 7\n",
        "TRAIN: [ 0  1  2  3  4  5  6  7  8  9 10 11 12 13 16 17 18 19]\n",
        "TEST: [14 15]\n",
        "KNN prediction [2 2]\n",
        "Fold: 8\n",
        "TRAIN: [ 0  1  2  3  4  5  6  7  8  9 10 11 12 13 14 15 18 19]\n",
        "TEST: [16 17]\n",
        "KNN prediction [2 2]\n",
        "Fold: 9\n",
        "TRAIN: [ 0  1  2  3  4  5  6  7  8  9 10 11 12 13 14 15 16 17]\n",
        "TEST: [18 19]\n",
        "KNN prediction [2 2]\n",
        "cross validation error: 0.000000\n",
        "cross validation accuracy: 1.000000\n",
        "None\n"
       ]
      }
     ],
     "prompt_number": 7
    },
    {
     "cell_type": "markdown",
     "metadata": {},
     "source": [
      "This code is also posted as a template in `sources/crossValidationTemplate.py`  "
     ]
    }
   ],
   "metadata": {}
  }
 ]
}