Test

.gitignore

@@ -1,3 +1,4 @@
+.idea
 code/*.pyc
 code/*_plots
 code/tmp*
@@ -13,3 +14,5 @@ html
 *.pyc
 *~
 *.swp
+# This directory may be created by scripts from segmentation tutorials.
+save_models

.jenkins/jenkins_buildbot_dlt.sh

@@ -1,18 +1,62 @@
 #!/bin/bash
 
+# CUDA
+export PATH=/usr/local/cuda/bin:$PATH
+export LD_LIBRARY_PATH=/usr/local/cuda/lib64:$LD_LIBRARY_PATH
+export LIBRARY_PATH=/usr/local/cuda/lib64:$LIBRARY_PATH
+
+# MKL
+export MKL_THREADING_LAYER=GNU
+
+# Set OpenMP threads for stability of speedtests
+export OMP_NUM_THREADS=1
+
 BUILDBOT_DIR=$WORKSPACE/nightly_build
-source $HOME/.bashrc
 
 mkdir -p ${BUILDBOT_DIR}
 
 date
-COMPILEDIR=$WORKSPACE/compile/lisa_theano_compile_dir_deeplearning
+COMPILEDIR=$HOME/.theano/lisa_theano_buildbot_deeplearning
 NOSETESTS=${BUILDBOT_DIR}/Theano/bin/theano-nose
 XUNIT="--with-xunit --xunit-file="
+# name test suites
+SUITE="--xunit-testsuite-name="
 
 FLAGS=warn.ignore_bug_before=0.5,compiledir=${COMPILEDIR}
 export PYTHONPATH=${BUILDBOT_DIR}/Theano:${BUILDBOT_DIR}/Pylearn:$PYTHONPATH
 
+# Install libgpuarray and pygpu
+cd ${BUILDBOT_DIR}
+
+# Make fresh clone (with no history since we don't need it)
+rm -rf libgpuarray
+git clone "https://github.com/Theano/libgpuarray.git"
+
+(cd libgpuarray && echo "libgpuarray commit" && git rev-parse HEAD)
+
+# Clean up previous installs (to make sure no old files are left)
+rm -rf local
+mkdir local
+
+# Build libgpuarray and run C tests
+mkdir libgpuarray/build
+(cd libgpuarray/build && cmake .. -DCMAKE_BUILD_TYPE=${GPUARRAY_CONFIG} -DCMAKE_INSTALL_PREFIX=${BUILDBOT_DIR}/local && make)
+
+# Finally install
+(cd libgpuarray/build && make install)
+export LD_LIBRARY_PATH=${BUILDBOT_DIR}/local/lib:${LD_LIBRARY_PATH}
+export LIBRARY_PATH=${BUILDBOT_DIR}/local/lib:${LIBRARY_PATH}
+export CPATH=${BUILDBOT_DIR}/local/include:${CPATH}
+
+# Build the pygpu modules
+(cd libgpuarray && python setup.py build_ext --inplace -I${BUILDBOT_DIR}/local/include -L${BUILDBOT_DIR}/local/lib)
+
+mkdir ${BUILDBOT_DIR}/local/lib/python
+export PYTHONPATH=${PYTHONPATH}:${BUILDBOT_DIR}/local/lib/python
+# Then install
+(cd libgpuarray && python setup.py install --home=${BUILDBOT_DIR}/local)
+
+# Install Theano
 cd ${BUILDBOT_DIR}
 if [ ! -d ${BUILDBOT_DIR}/Theano ]; then
   git clone git://github.com/Theano/Theano.git
@@ -28,9 +72,17 @@ echo "git version for Theano:" `git rev-parse HEAD`
 cd ${WORKSPACE}/code
 echo "git version:" `git rev-parse HEAD`
 
-echo "executing nosetests speed with mode=FAST_RUN"
-FILE=${BUILDBOT_DIR}/dlt_tests.xml
-THEANO_FLAGS=${FLAGS},mode=FAST_RUN ${NOSETESTS} ${XUNIT}${FILE} test.py:speed
-echo "executing nosetests with mode=FAST_RUN,floatX=float32"
-FILE=${BUILDBOT_DIR}/dlt_float32_tests.xml
-THEANO_FLAGS=${FLAGS},mode=FAST_RUN,floatX=float32 ${NOSETESTS} ${XUNIT}${FILE}
+echo "==== Executing nosetests speed with mode=FAST_RUN"
+NAME=dlt_speed
+FILE=${BUILDBOT_DIR}/${NAME}_tests.xml
+THEANO_FLAGS=${FLAGS},mode=FAST_RUN ${NOSETESTS} ${XUNIT}${FILE} ${SUITE}${NAME} test.py:speed
+
+echo "==== Executing nosetests with mode=FAST_RUN,floatX=float32"
+NAME=dlt_float32
+FILE=${BUILDBOT_DIR}/${NAME}_tests.xml
+THEANO_FLAGS=${FLAGS},mode=FAST_RUN,floatX=float32 ${NOSETESTS} ${XUNIT}${FILE} ${SUITE}${NAME}
+
+echo "==== Executing nosetests with mode=FAST_RUN,floatX=float32,device=cuda"
+NAME=dlt_float32_cuda
+FILE=${BUILDBOT_DIR}/${NAME}_tests.xml
+PYTHONPATH=${BUILDBOT_DIR}/Theano:${BUILDBOT_DIR}/DeepLearningTutorials/code:${PYTHONPATH} THEANO_FLAGS=${FLAGS},mode=FAST_RUN,floatX=float32,device=cuda nosetests test.py ${XUNIT}${FILE} ${SUITE}${NAME}

.travis.yml

@@ -78,6 +78,7 @@ script:
   - pwd
   - ls
   - export THEANO_FLAGS=warn.ignore_bug_before=all,on_opt_error=raise,on_shape_error=raise
+  - export MKL_THREADING_LAYER=GNU
   - python --version
   - nosetests -v $PART
 

README.rst

@@ -37,4 +37,4 @@ Subdirectories:
 Build instructions
 ------------------
 
-To build the html version of the tutorials, install sphinx and run doc/Makefile
+To build the html version of the tutorials, run python doc/scripts/docgen.py

code/DBN.py

@@ -340,7 +340,7 @@ def test_DBN(finetune_lr=0.1, pretraining_epochs=100,
                 c.append(pretraining_fns[i](index=batch_index,
                                             lr=pretrain_lr))
             print('Pre-training layer %i, epoch %d, cost ' % (i, epoch), end=' ')
-            print(numpy.mean(c))
+            print(numpy.mean(c, dtype='float64'))
 
     end_time = timeit.default_timer()
     # end-snippet-2
@@ -391,7 +391,7 @@ def test_DBN(finetune_lr=0.1, pretraining_epochs=100,
             if (iter + 1) % validation_frequency == 0:
 
                 validation_losses = validate_model()
-                this_validation_loss = numpy.mean(validation_losses)
+                this_validation_loss = numpy.mean(validation_losses, dtype='float64')
                 print('epoch %i, minibatch %i/%i, validation error %f %%' % (
                     epoch,
                     minibatch_index + 1,
@@ -414,7 +414,7 @@ def test_DBN(finetune_lr=0.1, pretraining_epochs=100,
 
                     # test it on the test set
                     test_losses = test_model()
-                    test_score = numpy.mean(test_losses)
+                    test_score = numpy.mean(test_losses, dtype='float64')
                     print(('     epoch %i, minibatch %i/%i, test error of '
                            'best model %f %%') %
                           (epoch, minibatch_index + 1, n_train_batches,

code/SdA.py

@@ -40,7 +40,7 @@
 
 import theano
 import theano.tensor as T
-from theano.tensor.shared_randomstreams import RandomStreams
+from theano.sandbox.rng_mrg import MRG_RandomStreams as RandomStreams
 
 from logistic_sgd import LogisticRegression, load_data
 from mlp import HiddenLayer
@@ -394,7 +394,7 @@ def test_SdA(finetune_lr=0.1, pretraining_epochs=15,
                 c.append(pretraining_fns[i](index=batch_index,
                          corruption=corruption_levels[i],
                          lr=pretrain_lr))
-            print('Pre-training layer %i, epoch %d, cost %f' % (i, epoch, numpy.mean(c)))
+            print('Pre-training layer %i, epoch %d, cost %f' % (i, epoch, numpy.mean(c, dtype='float64')))
 
     end_time = timeit.default_timer()
 
@@ -442,7 +442,7 @@ def test_SdA(finetune_lr=0.1, pretraining_epochs=15,
 
             if (iter + 1) % validation_frequency == 0:
                 validation_losses = validate_model()
-                this_validation_loss = numpy.mean(validation_losses)
+                this_validation_loss = numpy.mean(validation_losses, dtype='float64')
                 print('epoch %i, minibatch %i/%i, validation error %f %%' %
                       (epoch, minibatch_index + 1, n_train_batches,
                        this_validation_loss * 100.))
@@ -463,7 +463,7 @@ def test_SdA(finetune_lr=0.1, pretraining_epochs=15,
 
                     # test it on the test set
                     test_losses = test_model()
-                    test_score = numpy.mean(test_losses)
+                    test_score = numpy.mean(test_losses, dtype='float64')
                     print(('     epoch %i, minibatch %i/%i, test error of '
                            'best model %f %%') %
                           (epoch, minibatch_index + 1, n_train_batches,

code/cnn_1D_segm/data_loader/cortical_layers.py

@@ -0,0 +1,185 @@
+import os
+import time
+
+import numpy as np
+from PIL import Image
+import re
+import warnings
+
+from dataset_loaders.parallel_loader import ThreadedDataset
+from parallel_loader_1D import ThreadedDataset_1D
+
+floatX = 'float32'
+
+class Cortical6LayersDataset(ThreadedDataset_1D):
+    '''The Cortical Layers Dataset.
+    Parameters
+    ----------
+    which_set: string
+        A string in ['train', 'val', 'valid', 'test'], corresponding to
+        the set to be returned.
+    split: float
+        A float indicating the dataset split between training and validation.
+        For example, if split=0.85, 85\% of the images will be used for training,
+        whereas 15\% will be used for validation.
+    '''
+    name = 'cortical_layers'
+
+    non_void_nclasses = 7
+    GTclasses = [0, 1, 2, 3, 4, 5, 6]
+    _cmap = {
+        0: (128, 128, 128),    # padding
+        1: (128, 0, 0),        # layer 1
+        2: (128, 64, ),        # layer 2
+        3: (128, 64, 128),     # layer 3
+        4: (0, 0, 128),        # layer 4
+        5: (0, 0, 64),         # layer 5
+        6: (64, 64, 128),      # layer 6
+    }
+    _mask_labels = {0: 'padding', 1: 'layers1', 2: 'layer2', 3: 'layer3',
+                    4: 'layer4', 5: 'layer5',   6: 'layer6'}
+    _void_labels = []
+
+
+    _filenames = None
+
+    @property
+    def filenames(self):
+
+        if self._filenames is None:
+            # Load filenames
+            nfiles = sum(1 for line in open(self.mask_path))
+            filenames = range(nfiles)
+            np.random.seed(1609)
+            np.random.shuffle(filenames)
+
+            if self.which_set == 'train':
+                filenames = filenames[:int(nfiles*self.split)]
+            elif self.which_set == 'val':
+                filenames = filenames[-(nfiles - int(nfiles*self.split)):]
+
+            # Save the filenames list
+            self._filenames = filenames
+
+        return self._filenames
+
+    def __init__(self,
+                 which_set="train",
+                 split=0.85,
+                 shuffle_at_each_epoch = True,
+                 smooth_or_raw = 'both',
+                 *args, **kwargs):
+
+        self.task = 'segmentation'
+
+        self.n_layers = 6
+        n_layers_path = str(self.n_layers)+"layers_segmentation"
+
+        self.which_set = "val" if which_set == "valid" else which_set
+        if self.which_set not in ("train", "val", 'test'):
+            raise ValueError("Unknown argument to which_set %s" %
+                             self.which_set)
+
+        self.split = split
+
+        self.image_path_raw =  os.path.join(self.path,n_layers_path,"training_raw.txt")
+        self.image_path_smooth =  os.path.join(self.path,n_layers_path, "training_geo.txt")
+        self.mask_path = os.path.join(self.path,n_layers_path, "training_cls.txt")
+        self.regions_path = os.path.join(self.path, n_layers_path, "training_regions.txt")
+
+        self.smooth_raw_both = smooth_or_raw
+
+        if smooth_or_raw == 'both':
+            self.data_shape = (200,2)
+        else :
+            self.data_shape = (200,1)
+
+        super(Cortical6LayersDataset, self).__init__(*args, **kwargs)
+
+    def get_names(self):
+        """Return a dict of names, per prefix/subset."""
+
+        return {'default': self.filenames}
+
+
+
+def test_6layers():
+    train_iter = Cortical6LayersDataset(
+        which_set='train',
+        smooth_or_raw = 'both',
+        batch_size=500,
+        data_augm_kwargs={},
+        return_one_hot=False,
+        return_01c=False,
+        return_list=True,
+        use_threads=False)
+
+    valid_iter = Cortical6LayersDataset(
+        which_set='valid',
+        smooth_or_raw = 'smooth',
+        batch_size=500,
+        data_augm_kwargs={},
+        return_one_hot=False,
+        return_01c=False,
+        return_list=True,
+        use_threads=False)
+
+    valid_iter2 = Cortical6LayersDataset(
+        which_set='valid',
+        smooth_or_raw = 'raw',
+        batch_size=500,
+        data_augm_kwargs={},
+        return_one_hot=False,
+        return_01c=False,
+        return_list=True,
+        use_threads=False)
+
+
+
+    train_nsamples = train_iter.nsamples
+    train_nbatches = train_iter.nbatches
+    valid_nbatches = valid_iter.nbatches
+    valid_nbatches2 = valid_iter2.nbatches
+
+
+
+    # Simulate training
+    max_epochs = 1
+    print "Simulate training for", str(max_epochs), "epochs"
+    start_training = time.time()
+    for epoch in range(max_epochs):
+        print "Epoch #", str(epoch)
+
+        start_epoch = time.time()
+
+        print "Iterate on the training set", train_nbatches, "minibatches"
+        for mb in range(train_nbatches):
+            start_batch = time.time()
+            batch = train_iter.next()
+            if mb%5 ==0:
+                print("Minibatch train {}: {} sec".format(mb, (time.time() -
+                                                     start_batch)))
+
+        print "Iterate on the validation set", valid_nbatches, "minibatches"
+        for mb in range(valid_nbatches):
+            start_batch = time.time()
+            batch = valid_iter.next()
+            if mb%5 ==0:
+                print("Minibatch valid {}: {} sec".format(mb, (time.time() -
+                                                     start_batch)))
+
+        print "Iterate on the validation set (second time)", valid_nbatches2, "minibatches"
+        for mb in range(valid_nbatches2):
+            start_batch = time.time()
+            batch = valid_iter2.next()
+            if mb%5==0:
+                print("Minibatch valid {}: {} sec".format(mb, (time.time() -
+                                                     start_batch)))
+
+        print("Epoch time: %s" % str(time.time() - start_epoch))
+    print("Training time: %s" % str(time.time() - start_training))
+
+if __name__ == '__main__':
+    print "Loading the dataset 1 batch at a time"
+    test_6layers()
+    print "Success!"