Some problems were solved; some properties were added to a base class; attempt...

Some problems were solved; some properties were added to a base class; attempt to enable installation by means of pip

formats/asciigrid.py

@@ -3,6 +3,7 @@ import os.path
 import array
 from .raster import Raster
 from .gridenvelope2d import GridEnvelope2D;
+from builtins import int
 
 __author__ = "Steven B. Hoek"
 
@@ -204,4 +205,10 @@ class AsciiGrid(Raster, GridEnvelope2D):
         line = self.next()
         self.reset()
         return line[int(k)]
+    
+    def get_type(self):
+        if self.dataformat == 'i':
+            return int
+        else:
+            return float
         
\ No newline at end of file

formats/basetiffraster.py

@@ -25,7 +25,11 @@ class BaseTiffRaster(Raster):
     GeoAsciiParams = "GCS_WGS_1984|"
     GdalMetadata = '<GDALMetadata>\n  <Item name="RepresentationType" sample="0">ATHEMATIC</Item>\n</GDALMetadata>'
         
-    def __init__(self, filepath, *datatype):
+    def __init__(self, filepath='', *datatype):
+        # Check input
+        if filepath == '':
+            print('File path cannot be an empty string (method __init__).')
+            
         # Initialise
         super(BaseTiffRaster, self).__init__(filepath)
         

formats/geotiffraster.py

@@ -5,7 +5,7 @@ from formats.raster import Raster
 from math import fabs;
 from formats.const import constants as const
 import os;
-from tifffile import TiffFile;
+from tifffile import memmap, TiffFile;
 try:
     import numpy as np
     HAS_NUMPY = True
@@ -35,6 +35,8 @@ class GeotiffRaster(GridEnvelope2D, Raster):
     currow = -1;
     __envelope = None;
     __image = None;
+    __memmappable = False;
+    __memmap = None;
     
     def __init__(self, filepath, *datatype):
         # Retrieve the name from the filepath and assign - incl. extension
@@ -54,7 +56,10 @@ class GeotiffRaster(GridEnvelope2D, Raster):
         
         # If file does not exist and mode[0] = 'w', create it!
         if (mode[0] == 'w') and (not os.path.exists(self.folder + os.path.sep + self.name)):
-            self.datafile = open(self.folder + os.path.sep + self.name, 'w');
+            # We assume that no compression is needed
+            if self.__datatype == const.FLOAT: dtype = 'float32'
+            else: dtype = 'int32';
+            self.__memmap = np.memmap(self.folder + os.path.sep + self.name, dtype, mode='w+', shape=(nrows, ncols))
             self.__envelope = GridEnvelope2D.__init__(self, ncols, nrows, xll, yll, cellsize, cellsize);
             return True;
         else: 
@@ -83,30 +88,57 @@ class GeotiffRaster(GridEnvelope2D, Raster):
                     # Retrieve other ones from the TIFF file itself 
                     if self.datafile.byteorder == '<': self.byteorder = 'II';
                     else: self.byteorder = 'MM';
-                    self.__image = self.datafile.asarray();
-                    if len(self.__image.shape) > 3:
-                        raise ValueError("Not sure how to handle data with more than 3 dimensions!");
-                    axes = self.datafile.series[0]['axes'];
-                    posx = axes.index('X')
-                    posy = axes.index('Y')
-                    posc = 3 - posx - posy
-                    self.ncols = self.__image.shape[posx]; 
-                    self.nrows = self.__image.shape[posy];
-                    if HAS_NUMPY:
-                        self.__image = np.rollaxis(self.__image, posc, 1)
+                    
+                    # Get hold of the page 
+                    if len(self.datafile.pages) > 1:
+                        raise Warning('Unable to handle TIFF file with multiple pages!')
+                    page = self.datafile.pages[0];
+                    self.nrows = page.tags['ImageLength'].value;
+                    self.ncols = page.tags['ImageWidth'].value;
+                    try:
+                        page = self.datafile.pages[0];
+                        if 'GDAL_NODATA' in page.tags: self.nodatavalue = float(page.tags['GDAL_NODATA'].value);
+                    except: pass;
+                    
+                    # Use memmap where possible, i.e. when ismemmappable is true
+                    series = None
+                    offset = None 
+                    if len(self.datafile.series) == 0: raise Exception("No series found in TIFF file!")
+                    else:
+                        series = self.datafile.series[0]
+                        offset = series.offset
+                    self.__memmappable = page.is_memmappable or ((series is not None) and (offset is not None))
+                    if not self.__memmappable:
+                        self.__image = self.datafile.asarray();
+                        if len(self.__image.shape) > 3:
+                            raise ValueError("Not sure how to handle data with more than 3 dimensions!");
                     else:
-                        raise ImportError("Numpy not found on this system")
-                    msg = "Data not properly dimensioned"
-                    if not self.__image.shape[1] == self.nrows: raise Exception(msg)
-                    if not self.__image.shape[2] == self.ncols: raise Exception(msg)
+                        self.datafile.close();
+                        self.datafile = None;
+                        self.__memmap = memmap(self.folder + os.path.sep + self.name, mode='r', offset=0);
+
+                    # Check a few things, where appropriate
+                    if not self.__memmappable:
+                        #axes = self.datafile.series[0].axes;
+                        #posx = axes.index('X')
+                        #posy = axes.index('Y')
+                        #posc = 3 - posx - posy
+                        #self.ncols = self.__image.shape[posx]; 
+                        #self.nrows = self.__image.shape[posy];
+                        #if HAS_NUMPY:
+                        #    self.__image = np.rollaxis(self.__image, posc, 1)
+                        #else:
+                        #    raise ImportError("Numpy not found on this system")
+                        msg = "Data not properly dimensioned"
+                        if not self.__image.shape[0] == self.nrows: raise Exception(msg)
+                        if not self.__image.shape[1] == self.ncols: raise Exception(msg)
+                        
+                    # Calculate yll and find out the NODATA value
                     if self.ycoords_sort == 'DESC':
                         self.yll = self.yul - self.nrows * self.dy;
                     else:
                         self.yll = self.yul + self.nrows * self.dy; 
-                    try:
-                        page = self.datafile.pages[0];
-                        if 'gdal_nodata' in page.tags: self.nodatavalue = float(page.gdal_nodata);
-                    except: pass; 
+                    
                 self.__envelope = GridEnvelope2D.__init__(self, self.ncols, self.nrows, self.xll, self.yll, self.dx, self.dy);
                 return True;
             else: return False;
@@ -147,11 +179,15 @@ class GeotiffRaster(GridEnvelope2D, Raster):
         try:
             self.currow += 1;
             if (self.currow >= self.nrows): raise StopIteration;
-            if parseLine:
-                if len(self.__image.shape) == 1:
+            if self.__memmappable:
+                if parseLine:
+                    result = self.__memmap[self.currow];
+            else:
+                if parseLine:
+                    #if len(self.__image.shape) == 1:
                     result = self.__image[self.currow];  
-                else:
-                    result = self.__image[:, self.currow]
+                    #else:
+                    #   result = self.__image[:, self.currow]
             return result     
         except:
             raise StopIteration; 
@@ -185,7 +221,9 @@ class GeotiffRaster(GridEnvelope2D, Raster):
     def writenext(self, sequence_with_data):
         # Write the next data if possible, otherwise generate StopIteration
         # We cannot know whether exactly 1 row is included or not.
-        pass;
+        # If there's no need for compression, use memmap
+        self.__memmappable = True
+        
                 
     def reset(self):
         self.currow = -1;   
@@ -205,4 +243,8 @@ class GeotiffRaster(GridEnvelope2D, Raster):
             return result
         else:
             ImportError("Numpy not found on this system")
- 
\ No newline at end of file
+            
+    def close(self):
+        super(GeotiffRaster, self).close();
+        self.__memmap = None;
+              
\ No newline at end of file

formats/raster.py

@@ -60,5 +60,22 @@ class Raster(object):
 
     def reset(self):
         self.currow = 0; 
-    
+        
+    @property
+    def dx(self):
+        return self.cellsize
+
+    @dx.setter
+    def dx(self, dx):
+        self.cellsize = dx
+        
+    @property
+    def dy(self):
+        # TODO: differentiate dx and dy!
+        return self.cellsize
+
+    @dy.setter
+    def dy(self, dy):
+        # TODO: differentiate dx and dy!
+        self.cellsize = dy
     
\ No newline at end of file

formats/rowtiffraster.py

@@ -6,6 +6,10 @@ from libtiff import TIFF
 import numpy as np
 import os
 
+# TODO Python library pytiff may work better than libtiff. The problem is that it is difficult 
+# if not impossible at this moment to install / run it on Windows. Let's hope this will be solved
+# in the near future. If it is, then switch to pytiff.
+
 __author__ = "Steven B. Hoek"
 
 class RowTiffRaster(BaseTiffRaster, GridEnvelope2D):
@@ -35,7 +39,11 @@ class RowTiffRaster(BaseTiffRaster, GridEnvelope2D):
     __ReadStrip = dummy(0, 0, 1)
     __WriteStrip = dummy(0, 0, 1)
 
-    def __init__(self, filepath, *datatype):
+    def __init__(self, filepath='', *datatype):
+        # Check input
+        if filepath == '':
+            print('File path cannot be an empty string (method __init__).')
+            
         # Initialise
         super(RowTiffRaster, self).__init__(filepath)
         if self._const == None:

setup.py

+from __future__ import print_function
+from setuptools import setup, find_packages
+import os
+import io
+
+PACKAGE = "lmgeo"
+NAME = "Lmgeo"
+DESCRIPTION = 'Python raster GIS library with low memory requirements.' 
+
+AUTHOR = "Steven Hoek"
+AUTHOR_EMAIL = 'steven.hoek@wur.nl'
+URL = 'https://git.wur.nl/hoek008/lmgeo/'
+LICENSE="LGPL"
+VERSION = "1.0"
+
+here = os.path.abspath(os.path.dirname(__file__))
+
+def read(*filenames, **kwargs):
+    encoding = kwargs.get('encoding', 'utf-8')
+    sep = kwargs.get('sep', '\n')
+    buf = []
+    for filename in filenames:
+        with io.open(filename, encoding=encoding) as f:
+            buf.append(f.read())
+    return sep.join(buf)
+
+
+long_description = read('README.md')
+
+setup(
+    name=NAME,
+    version=VERSION,
+    url=URL,
+    download_url='https://git.wur.nl/hoek008/lmgeo/tarball/'+VERSION,
+    license='LGPL',
+    author=AUTHOR,
+    install_requires=['pyshp==2.1.0',
+                      'pyproj=1.9.5.1',
+                      'numpy==1.14.3',
+                      'tifffile==2019.3.18',
+                      'tables==3.5.2',
+                      'netCDF4==1.5.1.2',
+                      'libtiff==0.4.2']
+    author_email=AUTHOR_EMAIL,
+    description=DESCRIPTION,
+    long_description=long_description,
+    packages=find_packages(),
+    include_package_data=True,
+    platforms='any',
+    test_suite='lmgeo.tests.make_test_suite',
+    classifiers=[
+        'Development Status :: 4 - Beta',
+        'Environment :: Console',
+        'Intended Audience :: Developers',
+        'Intended Audience :: Science/Research',
+        'License :: OSI Approved :: GNU Lesser General Public License v3.0 (LGPL 3.0)',
+        'Natural Language :: English',
+        'Operating System :: OS Independent',
+        'Programming Language :: Python',
+        'Topic :: Scientific/Engineering']
+)
\ No newline at end of file

tests/__init__.py

@@ -4,6 +4,7 @@ from . import test_floatingpointraster
 from . import test_bilraster
 from . import test_bsqraster
 from . import test_csfraster
+#from . import test_rowtiffraster
 
 
 def make_test_suite(dsn=None):
@@ -13,7 +14,8 @@ def make_test_suite(dsn=None):
                                    test_floatingpointraster.suite(),
                                    test_bilraster.suite(),
                                    test_bsqraster.suite(),
-                                   test_csfraster.suite()
+                                   test_csfraster.suite(),
+                                   #test_rowtiffraster()
                                    ])
     return allsuites