Implementation of DataSet loading and saving (as a set of .hdf5 files)!

dataset: implemented save/load features in io_dataset.py
h5py: copied data from hdf5 files to make sure they exist after file is closed.
projector/phasemap/dataset: force dim/dim_uv to be tuples.

pyramid/dataset.py

@@ -119,6 +119,7 @@ class DataSet(object):
         return {kernel.dim_uv: PhaseMapperRDFC(kernel) for kernel in kernel_list}
 
     def __init__(self, a, dim, b_0=1, mask=None, Se_inv=None):
+        dim = tuple(dim)
         self._log.debug('Calling __init__')
         assert isinstance(dim, tuple) and len(dim) == 3, \
             'Dimension has to be a tuple of length 3!'
@@ -257,6 +258,20 @@ class DataSet(object):
                 mask_3d += projector.weight.T.dot(mask_2d).reshape(self.dim)
             self.mask = np.where(mask_3d >= threshold * self.count, True, False)
 
+    def save(self, filename, overwrite=True):
+        """Saves the dataset as a collection of HDF5 files.
+
+        Parameters
+        ----------
+        filename: str
+            Base name of the files which the dataset is saved into. HDF5 files are supported.
+        overwrite: bool, optional
+            If True (default), an existing file will be overwritten, if False, this
+            (silently!) does nothing.
+        """
+        from .file_io.io_dataset import save_dataset
+        save_dataset(self, filename, overwrite)
+
     def plot_mask(self, **kwargs):
         """If it exists, display the 3D mask of the magnetization distribution.
 
@@ -269,8 +284,7 @@ class DataSet(object):
         if self.mask is not None:
             return ScalarData(self.a, self.mask).plot_mask(**kwargs)
 
-    def plot_phasemaps(self, magdata=None, title='Phase Map',
-                       cmap='RdBu', limit=None, norm=None):
+    def plot_phasemaps(self, magdata=None, title='Phase Map', **kwargs):
         """Display all phasemaps saved in the :class:`~.DataSet` as a colormesh.
 
         Parameters
@@ -281,15 +295,6 @@ class DataSet(object):
         title : string, optional
             The main part of the title of the plots. The default is 'Phase Map'. Additional
             projector info is appended to this.
-        cmap : string, optional
-            The :class:`~matplotlib.colors.Colormap` which is used for the plots as a string.
-            The default is 'RdBu'.
-        limit : float, optional
-            Plotlimit for the phase in both negative and positive direction (symmetric around 0).
-            If not specified, the maximum amplitude of the phase is used.
-        norm : :class:`~matplotlib.colors.Normalize` or subclass, optional
-            Norm, which is used to determine the colors to encode the phase information.
-            If not specified, :class:`~matplotlib.colors.Normalize` is automatically used.
 
         Returns
         -------
@@ -301,8 +306,7 @@ class DataSet(object):
             phasemaps = self.create_phasemaps(magdata)
         else:
             phasemaps = self.phasemaps
-        [phasemap.plot_phase('{} ({})'.format(title, self.projectors[i].get_info()),
-                             cmap=cmap, limit=limit, norm=norm)
+        [phasemap.plot_phase('{} ({})'.format(title, self.projectors[i].get_info()), **kwargs)
          for (i, phasemap) in enumerate(phasemaps)]
 
     def plot_phasemaps_combined(self, magdata=None, title='Combined Plot', cmap='RdBu', limit=None,

pyramid/file_io/__init__.py

@@ -8,5 +8,6 @@ from .io_phasemap import load_phasemap
 from .io_vectordata import load_vectordata
 from .io_scalardata import load_scalardata
 from .io_projector import load_projector
+from .io_dataset import load_dataset
 
-__all__ = ['load_phasemap', 'load_vectordata', 'load_scalardata', 'load_projector']
+__all__ = ['load_phasemap', 'load_vectordata', 'load_scalardata', 'load_projector', 'load_dataset']

pyramid/file_io/io_dataset.py

+# -*- coding: utf-8 -*-
+# Copyright 2016 by Forschungszentrum Juelich GmbH
+# Author: J. Caron
+#
+"""IO functionality for DataSet objects."""
+
+import logging
+
+import os
+
+import h5py
+
+import numpy as np
+
+from ..dataset import DataSet
+from ..file_io.io_projector import load_projector
+from ..file_io.io_phasemap import load_phasemap
+
+__all__ = ['load_projector']
+_log = logging.getLogger(__name__)
+
+
+def save_dataset(dataset, filename, overwrite=True):
+    """%s"""
+    _log.debug('Calling save_dataset')
+    path, filename = os.path.split(filename)
+    name, extension = os.path.splitext(filename)
+    assert extension in ['.hdf5', ''], 'For now only HDF5 format is supported!'
+    filename = name + '.hdf5'  # In case no extension is provided, set to HDF5!
+    # Header file:
+    header_name = os.path.join(path, 'dataset_{}'.format(filename))
+    if not os.path.isfile(header_name) or overwrite:  # Write if file does not exist or if forced:
+        with h5py.File(header_name, 'w') as f:
+            f.attrs['a'] = dataset.a
+            f.attrs['dim'] = dataset.dim
+            f.attrs['b_0'] = dataset.b_0
+            if dataset.mask is not None:
+                f.create_dataset('mask', data=dataset.mask)
+            if dataset.Se_inv is not None:
+                f.create_dataset('Se_inv', data=dataset.Se_inv)
+    # PhaseMaps and Projectors:
+    for i, projector in enumerate(dataset.projectors):
+        projector_name = 'projector_{}_{}_{}{}'.format(name, i, projector.get_info(), extension)
+        projector.save(os.path.join(path, projector_name), overwrite=overwrite)
+        phasemap_name = 'phasemap_{}_{}_{}{}'.format(name, i, projector.get_info(), extension)
+        dataset.phasemaps[i].save(os.path.join(path, phasemap_name), overwrite=overwrite)
+save_dataset.__doc__ %= DataSet.save.__doc__
+
+
+def load_dataset(filename):
+    """Load HDF5 file into a :class:`~pyramid.dataset.DataSet` instance.
+
+    Parameters
+    ----------
+    filename:  str
+        The filename to be loaded.
+
+    Returns
+    -------
+    projector : :class:`~.Projector`
+        A :class:`~.Projector` object containing the loaded data.
+
+    Notes
+    -----
+    This loads a header file and all matching HDF5 files which can be found. The filename
+    conventions have to be strictly followed for the process to be successful!
+
+    """
+    _log.debug('Calling load_dataset')
+    path, filename = os.path.split(filename)
+    if path == '':
+        path = '.'  # Make sure this can be used later!
+    name, extension = os.path.splitext(filename)
+    assert extension in ['.hdf5', ''], 'For now only HDF5 format is supported!'
+    if name.startswith('dataset_'):
+        name = name.split('dataset_')[1]
+    filename = name + '.hdf5'  # In case no extension is provided, set to HDF5!
+    # Header file:
+    header_name = os.path.join(path, 'dataset_{}'.format(filename))
+    with h5py.File(header_name, 'r') as f:
+        a = f.attrs.get('a')
+        dim = f.attrs.get('dim')
+        b_0 = f.attrs.get('b_0')
+        mask = np.copy(f.get('mask', None))
+        Se_inv = np.copy(f.get('Se_inv', None))
+        dataset = DataSet(a, dim, b_0, mask, Se_inv)
+    # Projectors:
+    projectors = []
+    for f in os.listdir(path):
+        if f.startswith('projector') and f.endswith('.hdf5'):
+            projector_name, i = f.split('_')[1:3]
+            if projector_name == name:
+                projector = load_projector(os.path.join(path, f))
+                projectors.append((int(i), projector))
+    projectors = [p[1] for p in sorted(projectors, key=lambda x: x[0])]
+    dataset.projectors = projectors
+    # PhaseMaps:
+    phasemaps = []
+    for f in os.listdir(path):
+        if f.startswith('phasemap') and f.endswith('.hdf5'):
+            phasemap_name, i = f.split('_')[1:3]
+            if phasemap_name == name:
+                phasemap = load_phasemap(os.path.join(path, f))
+                phasemaps.append((int(i), phasemap))
+    phasemaps = [p[1] for p in sorted(phasemaps, key=lambda x: x[0])]
+    dataset.phasemaps = phasemaps
+    # Return DataSet:
+    return dataset

pyramid/file_io/io_projector.py

@@ -2,7 +2,7 @@
 # Copyright 2016 by Forschungszentrum Juelich GmbH
 # Author: J. Caron
 #
-"""IO functionality for FieldData objects."""
+"""IO functionality for Projector objects."""
 
 import logging
 
@@ -26,7 +26,7 @@ def save_projector(projector, filename, overwrite=True):
     name, extension = os.path.splitext(filename)
     assert extension in ['.hdf5', ''], 'For now only HDF5 format is supported!'
     filename = name + '.hdf5'  # In case no extension is provided, set to HDF5!
-    if not os.path.isfile(filename) or overwrite:  # Write if file does not exist of if forced:
+    if not os.path.isfile(filename) or overwrite:  # Write if file does not exist or if forced:
         with h5py.File(filename, 'w') as f:
             class_name = projector.__class__.__name__
             f.attrs['class'] = class_name
@@ -74,7 +74,7 @@ def load_projector(filename):
         indices = f.get('indices')
         weight = csr_matrix((data, indices, indptr), shape=(size_2d, size_3d))
         # Retrieve coefficients:
-        coeff = f.get('coeff')
+        coeff = np.copy(f.get('coeff'))
         # Construct projector:
         result = projector.Projector(dim, dim_uv, weight, coeff)
         # Specify projector type:

pyramid/phasemap.py

@@ -126,7 +126,7 @@ class PhaseMap(object):
     @phase.setter
     def phase(self, phase):
         assert isinstance(phase, np.ndarray), 'Phase has to be a numpy array!'
-        assert len(phase.shape) == 2, 'Phase has to be 2-dimensional!'
+        assert len(phase.shape) == 2, 'Phase has to be 2-dimensional, not {}!'.format(phase.shape)
         self._phase = phase.astype(dtype=np.float32)
         self._dim_uv = phase.shape
 

pyramid/projector.py

@@ -65,8 +65,8 @@ class Projector(object):
 
     def __init__(self, dim, dim_uv, weight, coeff):
         self._log.debug('Calling __init__')
-        self.dim = dim
-        self.dim_uv = dim_uv
+        self.dim = tuple(dim)
+        self.dim_uv = tuple(dim_uv)
         self.weight = weight
         self.coeff = coeff
         self.size_2d, self.size_3d = weight.shape
@@ -206,8 +206,7 @@ class Projector(object):
         Parameters
         ----------
         filename: str
-            Name of the file which the phasemap is saved into. The extension
-            determines the saving procedure.
+            Name of the file which the phasemap is saved into. HDF5 files are supported.
         overwrite: bool, optional
             If True (default), an existing file will be overwritten, if False, this
             (silently!) does nothing.