Merge branch 'deployment_improvements'

# Conflicts:
#	.gitlab-ci.yml

.gitignore

@@ -4,6 +4,8 @@ desktop.ini
 
 .idea
 
+.coverage
+
 *.egg-info
 
 .DS_Store

.gitlab-ci.yml

@@ -27,9 +27,9 @@ test:
     # TODO: Different jobs with custom develop arguments? extra_requires (hyperspy, plotting)?
     # TODO: Use pip install -e .[hyperspy], etc.
     # Install requirements (-e: editable, like python setup.py develop, but pip, not easy_install):
-    - pip install -e .  # TODO: belongs here or before_script?
+    - pip install -e .['tests']  # Also installs everything for tests!
     - python setup.py test
     - coverage html
   artifacts:
     paths:
-      - htmlcov/
\ No newline at end of file
+      - htmlcov/

environment.yml

+# Create environment with "conda env create", update with "conda env update"!
+
+name: pyramid
+
+channels:
+  - defaults
+  - conda-forge  # Used for hyperspy, pyFFTW (linux-64/osx-64) and more!
+  - ccpi  # Used for pyFFTW on win-64!
+
+dependencies:
+  # Basic:
+  - python=3.5  # TODO: upgrade to 3.6!
+  - numpy=1.11
+  - scipy=0.19
+  - tqdm=4.14
+  - scikit-image=0.13
+  # Fast computation:
+  - pyFFTW=0.10
+  # File IO:
+  - hyperspy=1.3
+  - hyperspy-gui-ipywidgets=1.0
+  - hyperspy-gui-traitsui=1.0  # TODO: Optional! Delete when mayavi is gone!
+  - h5py=2.7
+  # Plotting and colors:  # TODO: Shift to plotting package?
+  - matplotlib=2.0
+  - Pillow=4.1
+  - cmocean=1.1
+  - qt=4.8  # Necessary because of traits in mayavi...
+  - mayavi=4.5  # TODO: Get rid of!
+  #- colorspacious  # TODO: NOT USED?
+  #- visvis  # TODO: NOT USED?
+  # Testing:
+  - pytest=3.1
+  - pytest-cov=2.5
+  - pytest-flake8=0.8
+  - pytest-runner=2.11
+  - coverage=4.4
+  # IPython and notebooks:
+  - ipython=5.3
+  - jupyter=1.0
+  # TODO: Add back GUI dependencies!
+  # TODO: Get Jutil from gitlab (currently doesn't work, git and cygwin don't play nice,...
+  # TODO: ...because one is Unix, ond is Windows).
+#  - pip=9.0
+#  - pip:
+#    # ALSO NEEDS JUTIL!
+#    - "git+ssh://gitlab@iffgit.fz-juelich.de/unger/jutil.git"
+#
+#    # FOLLOWING NOT USED ANYMORE, CONDA WORKS, TOO (with appropriate channels):
+#    # Install pyFFTW=0.10 from wheel:
+#    - http://www.lfd.uci.edu/~gohlke/pythonlibs/pyFFTW‑0.10.4‑cp35‑cp35m‑win_amd64.whl
+#    # Install mayavi=4.5 and VTK from wheel:
+#    - http://www.lfd.uci.edu/~gohlke/pythonlibs/mayavi‑4.5.0+vtk71‑cp35‑cp35m‑win_amd64.whl

pyramid/fft.py

@@ -10,7 +10,7 @@ pack will be used. FFTW objects are saved in a cache after creation which speeds
 FFT operations.
 
 """
-
+# TODO: is this still in use or deprecated by jutil???
 
 import pickle
 import logging

pyramid/file_io/io_vectordata.py

@@ -155,7 +155,7 @@ def _load_from_ovf(filename, a):
         field = np.asarray((x_mag, y_mag, z_mag)) * float(header.get('valuemultiplier', 1))
         if a is None:
             # TODO: If transferred to HyperSpy, this has to stay in Pyramid reader!
-            if header.get('xstepsize') == header.get('ystepsize') == header.get('zstepsize'):
+            if not header.get('xstepsize') == header.get('ystepsize') == header.get('zstepsize'):
                 _log.warning('Grid spacing is not equal in x, y and z (x will be used)!')
             a = float(header.get('xstepsize', 1.))
             meshunit = header.get('meshunit', 'nm')

pyramid/tests/test_analytic.py

@@ -54,8 +54,3 @@ class TestCaseAnalytic(unittest.TestCase):
         reference = np.load(os.path.join(self.path, 'ref_phase_vort.npy'))
         assert_allclose(phase, reference, atol=1E-10,
                         err_msg='Unexpected behavior in phase_mag_vortex()')
-
-
-if __name__ == '__main__':
-    import nose
-    nose.run(defaultTest=__name__)

pyramid/tests/test_costfunction.py

@@ -74,8 +74,3 @@ class TestCaseCostfunction(unittest.TestCase):
     def test_hess_diag(self):
         assert_allclose(self.cost.hess_diag(None), np.ones(self.cost.n),
                         err_msg='Unexpected behaviour in hess_diag()!')
-
-
-if __name__ == '__main__':
-    import nose
-    nose.run(defaultTest=__name__)

pyramid/tests/test_dataset.py

@@ -88,8 +88,3 @@ class TestCaseDataSet(unittest.TestCase):
         mask_ref[1:-1, 1:-1, 1:-1] = True
         np.testing.assert_equal(self.data.mask, mask_ref,
                                 err_msg='Unexpected behaviour in set_3d_mask')
-
-
-if __name__ == '__main__':
-    import nose
-    nose.run(defaultTest=__name__)

pyramid/tests/test_fielddata.py

@@ -116,8 +116,3 @@ class TestCaseVectorData(unittest.TestCase):
                         err_msg='Unexpected behavior in load_from_hdf5()!')
         assert_allclose(magdata.a, self.magdata.a,
                         err_msg='Unexpected behavior in load_from_hdf5()!')
-
-
-if __name__ == '__main__':
-    import nose
-    nose.run(defaultTest=__name__)

pyramid/tests/test_forwardmodel.py

@@ -67,8 +67,3 @@ class TestCaseForwardModel(unittest.TestCase):
         jac_T_ref = np.load(os.path.join(self.path, 'jac.npy')).T
         assert_allclose(jac_T, jac_T_ref, atol=1E-7,
                         err_msg='Unexpected behaviour in the the transposed jacobi matrix!')
-
-
-if __name__ == '__main__':
-    import nose
-    nose.run(defaultTest=__name__)

pyramid/tests/test_kernel.py

@@ -30,8 +30,3 @@ class TestCaseKernel(unittest.TestCase):
                         err_msg='Unexpected behavior in u_fft')
         assert_allclose(self.kernel.v_fft, ref_v_fft, atol=1E-7,
                         err_msg='Unexpected behavior in v_fft')
-
-
-if __name__ == '__main__':
-    import nose
-    nose.run(defaultTest=__name__)

pyramid/tests/test_magcreator.py

@@ -78,8 +78,3 @@ class TestCaseMagCreator(unittest.TestCase):
         magnitude = mc.create_mag_dist_vortex(mag_shape)
         assert_allclose(magnitude, np.load(os.path.join(self.path, 'ref_mag_vort.npy')),
                         err_msg='Created vortex magnetic distribution does not match expectation')
-
-
-if __name__ == '__main__':
-    import nose
-    nose.run(defaultTest=__name__)

pyramid/tests/test_phasemap.py

@@ -72,8 +72,3 @@ class TestCasePhaseMap(unittest.TestCase):
                         err_msg='Unexpected behavior in load_from_netcdf4()!')
         assert_allclose(phasemap.a, self.phasemap.a,
                         err_msg='Unexpected behavior in load_from_netcdf4()!')
-
-
-if __name__ == '__main__':
-    import nose
-    nose.run(defaultTest=__name__)

pyramid/tests/test_phasemapper.py

@@ -201,8 +201,3 @@ class TestCasePhaseMapperCharge(unittest.TestCase):
 
     def test_jac_T_dot(self):
         self.assertRaises(NotImplementedError, self.mapper.jac_T_dot, None)
-
-
-if __name__ == '__main__':
-    import nose
-    nose.run(defaultTest=__name__)

pyramid/tests/test_pm.py

@@ -26,8 +26,3 @@ class TestCasePM(unittest.TestCase):
         assert_allclose(phasemap.phase, phase_ref.phase, atol=1E-7,
                         err_msg='Unexpected behavior in pm()!')
         assert_allclose(phasemap.a, phase_ref.a, err_msg='Unexpected behavior in pm()!')
-
-
-if __name__ == '__main__':
-    import nose
-    nose.run(defaultTest=__name__)

pyramid/tests/test_projector.py

@@ -256,7 +256,3 @@ class TestCaseYTiltProjector(unittest.TestCase):
 
 
 # TODO: Test RotTiltProjector!!!
-
-if __name__ == '__main__':
-    import nose
-    nose.run(defaultTest=__name__)

pyramid/tests/test_regularisator.py

@@ -128,8 +128,3 @@ class TestCaseFirstOrderRegularisator(unittest.TestCase):
         hess_diag_ref = np.diag(np.load(os.path.join(self.path, 'first_order_jac_ref.npy')))
         assert_allclose(hess_diag, hess_diag_ref, atol=1E-7,
                         err_msg='Unexpected behaviour in hess_diag()!')
-
-
-if __name__ == '__main__':
-    import nose
-    nose.run(defaultTest=__name__)

requirements.txt

+# REQUIREMENTS FOR PYRAMID
+#
+# Use "pip install -f requirements.txt" to install with pip!
+# If you want to install basic dependencies according to setup.py use "pip install -e .['all']"!
+#
+# OR (recommended):
+#
+# Create environment with "conda create -n pyramid -f requirements.txt"!
+# Add necessary channels:
+#       - "conda-forge" --> hyperspy (win-32/64, linux-64, osx-64), pyFFTW (linux-64, osx-64)
+#       - "ccpi" --> pyFFTW (win-64)
+# with one of these options:
+#       - add them as a parameter to "conda create -n pyramid -f requirements.txt -c CHANNELS"
+#       - add them to ~/.condarc with "conda config --append channels CHANNEL" (one by one)
+#
+# The jutil package from "https://iffgit.fz-juelich.de/unger/jutil" has to be installed manually!
+
+python==3.5
+numpy==1.11
+scipy==0.19
+tqdm==4.14
+scikit-image==0.13
+pyfftw==0.10
+hyperspy==1.3
+hyperspy-gui-ipywidgets==1.0
+hyperspy-gui-traitsui==1.0
+h5py==2.7
+matplotlib==2.0
+Pillow==4.1
+cmocean==1.1
+qt==4.8
+mayavi==4.5
+pytest==3.1
+pytest-cov==2.5
+pytest-flake8==0.8
+pytest-runner==2.11
+coverage==4.4
+ipython==5.3
+jupyter==1.0

setup.cfg

@@ -36,7 +36,7 @@ omit =
 
 [tool:pytest]
 addopts = --cov --flake8
-flake8-max-line-length = 120
+flake8-max-line-length = 100
 flake8-ignore =
     ALL  # TODO: PEP8 deactivated by this line (remove at a later point)!
     E402 E124 E125

setup.py

@@ -6,9 +6,10 @@ import os
 import re
 import subprocess
 import sys
+import itertools
 from distutils.command.build import build
 
-import numpy
+#import numpy
 from setuptools import setup, find_packages
 
 
@@ -17,12 +18,13 @@ DESCRIPTION = 'PYthon based Reconstruction Algorithm for MagnetIc Distributions'
 MAINTAINER = 'Jan Caron'
 MAINTAINER_EMAIL = 'j.caron@fz-juelich.de'
 URL = ''
-VERSION = '0.1.0-dev'
-PYTHON_VERSION = (2, 7)
-DEPENDENCIES = {'numpy': (1, 10)}
-LONG_DESCRIPTION = 'long description (TODO!)'  # TODO: Long description!
+VERSION = '0.1.0.dev0'  # TODO: Better way?
+PYTHON_VERSION = (2, 7)  # TODO: get rid of!!!
+DEPENDENCIES = {'numpy': (1, 10)}  # TODO: get rid of!!!
+LONG_DESCRIPTION = 'long description (TODO!)'  # TODO: Long description! put in (Readme?) file!
 
 
+# TODO: get rid of superfluous functions!
 def get_package_version(package):
     """Return the package version of the specified package.
 
@@ -70,7 +72,7 @@ def check_requirements():
                               % ((package_name,) + min_version))
 
 
-def hg_version():  # TODO: Replace with GIT! Also check build output on GitLab!
+def hg_version():  # TODO: Replace with GIT! Also check build output on GitLab! See numpy setup.py!
     """Get the Mercurial reference identifier.
 
     Returns
@@ -126,6 +128,26 @@ def get_files(rootdir):
     return filepaths
 
 
+
+# TODO: Use requirements.txt? extras_require for optional stuff (hyperspy, plotting)?
+# TODO: After split of Pyramid, comment out and see what really is used (is e.g. scipy?)!
+install_requires = ['numpy>=1.6', 'tqdm', 'scipy', 'matplotlib', 'Pillow', 'h5py',
+                    'hyperspy', 'jutil', 'cmocean']
+
+# TODO: extend extras_require for plotting and IO:
+# TODO: extra: 'pyfftw', 'mayavi' (not easy to install... find a way!)
+# TODO: See https://stackoverflow.com/a/28842733 for extras_require...
+# TODO: ...replace [dev] with [IO] (hyperspy) and [plotting] (separate plotting library)!
+extras_require = {
+    # TODO: Test all if really needed! don't use nose, if possible (pure pytest)!
+    'tests': ['pytest', 'pytest-runner', 'pytest-cov', 'pytest-flake8', 'coverage'],
+    '3Dplot': ['qt==4.8', 'mayavi==4.5']
+    # TODO: more for mayavi (plotting in general) and hyperspy, etc (see below)...
+}
+
+extras_require["all"] = list(itertools.chain(*list(extras_require.values())))
+
+
 print('\n-------------------------------------------------------------------------------')
 print('checking requirements')
 check_requirements()
@@ -139,18 +161,12 @@ setup(name=DISTNAME,
       url=URL,
       download_url=URL,
       version=VERSION,
-      packages=find_packages(exclude=['tests', 'doc']),
-      include_dirs=[numpy.get_include()],
-      # TODO: Use requirements.txt? extras_require for optional stuff (hyperspy, plotting)?
-      # TODO: After split of Pyramid, comment out and see what really is used (is e.g. scipy?)!
-      # - pip install numpy scipy nose h5py matplotlib Pillow scikit-image cmocean hyperspy
-      setup_requires=['numpy>=1.6', 'pytest-runner', 'pytest'],
-      tests_require=['nose', 'pytest-cov', 'pytest-flake8', 'coverage'],
-      install_requires=['numpy>=1.6', 'tqdm', 'scipy', 'matplotlib', 'Pillow', 'h5py',
-                        'hyperspy', 'jutil', 'cmocean'],
-      # TODO: extra: 'pyfftw', 'mayavi' (not easy to install... find a way!)
-      # TODO: See https://stackoverflow.com/a/28842733 for extras_require...
-      # TODO: ...replace [dev] with [IO] (hyperspy) and [plotting] (separate plotting library)!
-      test_suite='nose.collector',
-      cmdclass={'build': build})
+      packages=find_packages(exclude=['tests', 'doc']),  # TODO: necessary?
+      #include_dirs=[numpy.get_include()],  # TODO: Maybe used for sphinx?!
+      #setup_requires=['numpy>=1.6', 'pytest', 'pytest-runner'],
+      #tests_require=['pytest', 'pytest-cov', 'pytest-flake8'],
+      install_requires=install_requires,
+      extras_require=extras_require,
+      cmdclass={'build': build}  # TODO: necessary?
+    )
 print('-------------------------------------------------------------------------------\n')