Merge branch 'master' of github.com:learnables/learn2learn

Author: seba-1511

CHANGELOG.md

@@ -10,15 +10,27 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ### Added
 
+### Changed
+
+### Fixed
+
+
+## v0.1.6
+
+### Added
+
 * PyTorch Lightning interface to MAML, ANIL, ProtoNet, MetaOptNet.
-* Automatic batcher for Lighting: `l2l.data.EpisodicBatcher`.
+* Automatic batcher for Lightning: `l2l.data.EpisodicBatcher`.
 * `l2l.nn.PrototypicalClassifier` and `l2l.nn.SVMClassifier`.
 * Add `l2l.vision.models.WRN28`.
 * Separate modules for `CNN4Backbone`, `ResNet12Backbone`, `WRN28Backbones` w/ pretrained weights.
 * Add `l2l.data.OnDeviceDataset` and implement `device` parameter for benchmarks.
+* (Beta) Add `l2l.data.partition_task` and `l2l.data.InfiniteIterator`.
 
 ### Changed
 
+* Renamed and clarify dropout parameters for `ResNet12`.
+
 ### Fixed
 
 * Improved support for 1D inputs in `l2l.nn.KroneckerLinear`. (@timweiland)

docs/pydocmd.yml

@@ -25,6 +25,10 @@ generate:
               - learn2learn.data.transforms.FusedNWaysKShots
               - learn2learn.data.transforms.RemapLabels
               - learn2learn.data.transforms.ConsecutiveLabels
+        - learn2learn.data.utils:
+              - learn2learn.data.utils.OnDeviceDataset
+              - learn2learn.data.utils.InfiniteIterator
+              - learn2learn.data.utils.partition_task
   - docs/learn2learn.algorithms.md:
       - learn2learn.algorithms:
           - learn2learn.algorithms.MAML++

learn2learn/_version.py

@@ -1 +1 @@
-__version__ = '0.1.5'
+__version__ = '0.1.6'

learn2learn/data/utils.py

@@ -49,6 +49,26 @@ def save_response_content(response, destination):
 
 class InfiniteIterator(object):
 
+    """
+    [[Source]](https://github.com/learnables/learn2learn/blob/master/learn2learn/data/utils.py)
+
+    **Description**
+
+    Infinitely loops over a given iterator.
+
+    **Arguments**
+
+    * **dataloader** (iterator) - Iterator to loop over.
+
+    **Example**
+    ~~~python
+    dataloader = DataLoader(dataset, shuffle=True, batch_size=32)
+    inf_dataloader = InfiniteIterator(dataloader)
+    for iteration in range(10000):  # guaranteed to reach 10,000 regardless of len(dataloader)
+        X, y = next(inf_dataloader)
+    ~~~
+    """
+
     def __init__(self, dataloader):
         self.dataloader = dataloader
         self.iterator = iter(self.dataloader)
@@ -64,11 +84,35 @@ def __next__(self):
                 self.iterator = iter(self.dataloader)
 
 
-def partition_task(data, labels, shots=1, ways=None):
+def partition_task(data, labels, shots=1):
+
+    """
+    [[Source]](https://github.com/learnables/learn2learn/blob/master/learn2learn/data/utils.py)
+
+    **Description**
+
+    Partitions a classification task into support and query sets.
+
+    The support set will contain `shots` samples per class, the query will take the remaining samples.
+
+    Assumes each class in `labels` is associated with the same number of samples in `data`.
+
+    **Arguments**
+
+    * **data** (Tensor) - Data to be partitioned into support and query.
+    * **labels** (Tensor) - Labels of each data sample, used for partitioning.
+    * **shots** (int, *optional*, default=1) - Number of data samples per class in the support set.
+
+    **Example**
+    ~~~python
+    X, y = taskset.sample()
+    (X_support, y_support), (X_query, y_query) = partition_task(X, y, shots=5)
+    ~~~
+    """
+
     assert data.size(0) == labels.size(0)
     unique_labels = labels.unique()
-    if ways is None:
-        ways = unique_labels.numel()
+    ways = unique_labels.numel()
     data_shape = data.shape[1:]
     num_support = ways * shots
     num_query = data.size(0) - num_support
@@ -119,6 +163,35 @@ def partition_task(data, labels, shots=1, ways=None):
 
 class OnDeviceDataset(torch.utils.data.TensorDataset):
 
+    """
+    [[Source]](https://github.com/learnables/learn2learn/blob/master/learn2learn/data/utils.py)
+
+    **Description**
+
+    Converts an entire dataset into a TensorDataset, and optionally puts it on the desired device.
+
+    Useful to accelerate training with relatively small datasets.
+    If the device is cpu and cuda is available, the TensorDataset will live in pinned memory.
+
+    **Arguments**
+
+    * **dataset** (Dataset) - Dataset to put on a device.
+    * **device** (torch.device, *optional*, default=None) - Device of dataset. Defaults to CPU.
+    * **transform** (transform, *optional*, default=None) - Transform to apply on the first variate of the dataset's samples X.
+
+    **Example**
+    ~~~python
+    transforms = transforms.Compose([
+        transforms.ToTensor(),
+        transforms.Normalize((0.1307,), (0.3081,)),
+        lambda x: x.view(1, 28, 28),
+    ])
+    mnist = MNIST('~/data')
+    mnist_ondevice = OnDeviceDataset(mnist, device='cuda', transform=transforms)
+    mnist_meta = MetaDataset(mnist_ondevice)
+    ~~~
+    """
+
     def __init__(self, dataset, device=None, transform=None):
         data = []
         labels = []
@@ -130,6 +203,9 @@ def __init__(self, dataset, device=None, transform=None):
         if device is not None:
             data = data.to(device)
             labels = labels.to(device)
+        if data.device == torch.device('cpu') and torch.cuda.is_available():
+            data = data.pin_memory()
+            labels = labels.pin_memory()
         super(OnDeviceDataset, self).__init__(data, labels)
         self.transform = transform
         if hasattr(dataset, '_bookkeeping_path'):

learn2learn/vision/models/cnn4.py

@@ -227,6 +227,22 @@ def forward(self, x):
 
 class CNN4Backbone(ConvBase):
 
+    def __init__(
+        self,
+        hidden_size=64,
+        layers=4,
+        channels=3,
+        max_pool=False,
+        max_pool_factor=1.0,
+    ):
+        super(CNN4Backbone, self).__init__(
+            hidden=hidden_size,
+            layers=layers,
+            channels=channels,
+            max_pool=max_pool,
+            max_pool_factor=max_pool_factor,
+        )
+
     def forward(self, x):
         x = super(CNN4Backbone, self).forward(x)
         x = x.reshape(x.size(0), -1)
@@ -265,18 +281,19 @@ class CNN4(torch.nn.Module):
     def __init__(
         self,
         output_size,
-        hidden_size=32,
+        hidden_size=64,
         layers=4,
         channels=3,
+        max_pool=True,
         embedding_size=None,
     ):
         super(CNN4, self).__init__()
         if embedding_size is None:
             embedding_size = 25 * hidden_size
         self.features = CNN4Backbone(
-            hidden=hidden_size,
+            hidden_size=hidden_size,
             channels=channels,
-            max_pool=True,
+            max_pool=max_pool,
             layers=layers,
             max_pool_factor=4 // layers,
         )

learn2learn/vision/models/resnet12.py

@@ -1,6 +1,5 @@
 #!/usr/bin/env python3
 
-import os
 import torch
 import torch.nn as nn
 import torch.nn.functional as F
@@ -167,10 +166,10 @@ class ResNet12Backbone(nn.Module):
 
     def __init__(
         self,
-        keep_prob=1.0,  # dropout for embedding
         avg_pool=True,  # Set to False for 16000-dim embeddings
         wider=True,  # True mimics MetaOptNet, False mimics TADAM
-        drop_rate=0.1,  # dropout for residual layers
+        embedding_dropout=0.0,  # dropout for embedding
+        dropblock_dropout=0.1,  # dropout for residual layers
         dropblock_size=5,
         channels=3,
     ):
@@ -186,27 +185,27 @@ def __init__(
             block,
             num_filters[0],
             stride=2,
-            drop_rate=drop_rate,
+            dropblock_dropout=dropblock_dropout,
         )
         self.layer2 = self._make_layer(
             block,
             num_filters[1],
             stride=2,
-            drop_rate=drop_rate,
+            dropblock_dropout=dropblock_dropout,
         )
         self.layer3 = self._make_layer(
             block,
             num_filters[2],
             stride=2,
-            drop_rate=drop_rate,
+            dropblock_dropout=dropblock_dropout,
             drop_block=True,
             block_size=dropblock_size,
         )
         self.layer4 = self._make_layer(
             block,
             num_filters[3],
             stride=2,
-            drop_rate=drop_rate,
+            dropblock_dropout=dropblock_dropout,
             drop_block=True,
             block_size=dropblock_size,
         )
@@ -215,10 +214,10 @@ def __init__(
         else:
             self.avgpool = l2l.nn.Lambda(lambda x: x)
         self.flatten = l2l.nn.Flatten()
-        self.keep_prob = keep_prob
+        self.embedding_dropout = embedding_dropout
         self.keep_avg_pool = avg_pool
-        self.dropout = nn.Dropout(p=1.0 - self.keep_prob, inplace=False)
-        self.drop_rate = drop_rate
+        self.dropout = nn.Dropout(p=self.embedding_dropout, inplace=False)
+        self.dropblock_dropout = dropblock_dropout
 
         for m in self.modules():
             if isinstance(m, nn.Conv2d):
@@ -236,7 +235,7 @@ def _make_layer(
         block,
         planes,
         stride=1,
-        drop_rate=0.0,
+        dropblock_dropout=0.0,
         drop_block=False,
         block_size=1,
     ):
@@ -253,7 +252,7 @@ def _make_layer(
             planes,
             stride,
             downsample,
-            drop_rate,
+            dropblock_dropout,
             drop_block,
             block_size)
         )
@@ -301,14 +300,14 @@ class ResNet12(nn.Module):
 
     **Arguments**
 
-    * **output_size** (int) - The dimensionality of the output.
+    * **output_size** (int) - The dimensionality of the output (eg, number of classes).
     * **hidden_size** (list, *optional*, default=640) - Size of the embedding once features are extracted.
         (640 is for mini-ImageNet; used for the classifier layer)
-    * **keep_prob** (float, *optional*, default=1.0) - Dropout rate on the embedding layer.
     * **avg_pool** (bool, *optional*, default=True) - Set to False for the 16k-dim embeddings of Lee et al, 2019.
     * **wider** (bool, *optional*, default=True) - True uses (64, 160, 320, 640) filters akin to Lee et al, 2019.
         False uses (64, 128, 256, 512) filters, akin to Oreshkin et al, 2018.
-    * **drop_rate** (float, *optional*, default=0.1) - Dropout rate for the residual layers.
+    * **embedding_dropout** (float, *optional*, default=0.0) - Dropout rate on the flattened embedding layer.
+    * **dropblock_dropout** (float, *optional*, default=0.1) - Dropout rate for the residual layers.
     * **dropblock_size** (int, *optional*, default=5) - Size of drop blocks.
 
     **Example**
@@ -321,19 +320,19 @@ def __init__(
         self,
         output_size,
         hidden_size=640,  # mini-ImageNet images, used for the classifier
-        keep_prob=1.0,  # dropout for embedding
         avg_pool=True,  # Set to False for 16000-dim embeddings
         wider=True,  # True mimics MetaOptNet, False mimics TADAM
-        drop_rate=0.1,  # dropout for residual layers
+        embedding_dropout=0.0,  # dropout for embedding
+        dropblock_dropout=0.1,  # dropout for residual layers
         dropblock_size=5,
         channels=3,
     ):
         super(ResNet12, self).__init__()
         self.features = ResNet12Backbone(
-            keep_prob=keep_prob,
             avg_pool=avg_pool,
             wider=wider,
-            drop_rate=drop_rate,
+            embedding_dropout=embedding_dropout,
+            dropblock_dropout=dropblock_dropout,
             dropblock_size=dropblock_size,
             channels=channels,
         )
@@ -346,10 +345,9 @@ def forward(self, x):
 
 
 if __name__ == '__main__':
-    model = ResNet12(output_size=5, avg_pool=False, drop_rate=0.0)
+    model = ResNet12(output_size=5, avg_pool=False, dropblock_dropout=0.0)
     img = torch.randn(5, 3, 84, 84)
     model = model.to('cuda')
     img = img.to('cuda')
     out = model.features(img)
     print(out.shape)
-    __import__('pdb').set_trace()

setup.py

@@ -42,10 +42,11 @@
 ]
 
 if use_cython:
-    compiler_directives = {'language_level': 3,
-                           'embedsignature': True,
-    #                       'profile': True,
-    #                       'binding': True,
+    compiler_directives = {
+        'language_level': 3,
+        'embedsignature': True,
+        #  'profile': True,
+        #  'binding': True,
     }
     extensions = cythonize(extensions, compiler_directives=compiler_directives)