New k8s implementation

.python-version

+3.9.5

.s2i/environment

-APP_MODULE=appWSM:app

Dockerfile

-FROM ubuntu:18.04
-LABEL MAINTAINER Luís de Sousa <luis.desousa@wur.nl>
+FROM python:3.9.5-slim-buster@sha256:076f9edf940d59446b98c242de0058054240b890a47d1dbed18560d12ec58228
 
-RUN apt update && apt -y install python-pip python-dev nginx curl && rm -rf /var/lib/apt/lists/*
-RUN mkdir -p /var/www/wsm.rest
+# Required volume for monoliths e.g:
+# docker volume create monoliths-volume
+# docker build -t wsm.rest
+# docker run -it -v /data/monoliths-volume:/data/monoliths .
 
-WORKDIR /var/www/wsm.rest
+LABEL "maintainer"="jorge.mendesdejesus@isric.org"
+
+ENV LANG=C.UTF-8 LC_ALL=C.UTF-8
+ENV DEBIAN_FRONTEND noninteractive
+
+ENV DB_HOST scomp1270.wurnet.nl
+ENV DB_NAME isric
+ENV DB_USER hass005
+ENV DB_PASSWORD 1515NewPW
+ENV DB_PORT 5479
+ENV MONOLITH_PATH /data/monoliths
+
+RUN apt-get update \
+      && apt-get install -y --no-install-recommends postgresql-client build-essential libpq-dev tini
+
+RUN mkdir -p /app
+# Mount point with data
+RUN mkdir -p  /data/monoliths
+WORKDIR /app
 
 COPY . .
 
-RUN chmod +x configs/startup.sh
 RUN pip install -r requirements.txt
 
-ENTRYPOINT [ "configs/startup.sh" ]
+USER www-data
+EXPOSE 8000
+
+ENTRYPOINT ["/usr/bin/tini","-g","--"]
+CMD ["gunicorn","--bind","0.0.0.0","--workers","2","--threads","4","--worker-class","eventlet", "--log-file=-", "wsgi:app"]
+
+#         build-essential \
+
+#RUN apt-get update \
+#      && apt-get install -y --no-install-recommends \
+#         postgresql-client \
+#         build-essential \
+#         gdal-bin libgdal-dev libtbb2 tini 
+#WORKDIR /src
+#RUN mkdir -p /soils
+
+#COPY soils /soils/soils
+#COPY requirements.txt /soils/requirements.txt
+#COPY gunicorn.py /soils/gunicorn.py
+
+
+#WORKDIR /soils
+#RUN pip install -r requirements.txt
+
+# Clean up
+#RUN pip cache purge \ 
+#    && apt remove -y build-essential \ 
+#    && apt -y autoremove \ 
+#    && rm -rf /var/lib/apt/lists/*
+
+#USER www-data
+#EXPOSE 8000
+
+#ENTRYPOINT ["/usr/bin/tini","-g","--"]
+#CMD ["gunicorn","--config=./gunicorn.py", "soils:app"]
+
+
+#FROM ubuntu:18.04
+#LABEL MAINTAINER Luís de Sousa <luis.desousa@wur.nl>
+
+#RUN apt update && apt -y install python-pip python-dev nginx curl && rm -rf /var/lib/apt/lists/*
+#RUN mkdir -p /var/www/wsm.rest
+
+#WORKDIR /var/www/wsm.rest
+
+#COPY . .
+
+#RUN chmod +x configs/startup.sh
+#RUN pip install -r requirements.txt
+
+#ENTRYPOINT [ "configs/startup.sh" ]
 
-EXPOSE 80
-STOPSIGNAL SIGQUIT
+#EXPOSE 80
+#STOPSIGNAL SIGQUIT
 
-HEALTHCHECK CMD curl --fail http://localhost:80 || exit 1
-CMD ["nginx", "-g", "daemon off;"]
+#HEALTHCHECK CMD curl --fail http://localhost:80 || exit 1
+#CMD ["nginx", "-g", "daemon off;"]
 ################### Usage #####################
 
 # Build it
@@ -29,4 +97,4 @@ CMD ["nginx", "-g", "daemon off;"]
 # docker run -dit --env-file /path/to/envfile -v /data/monoliths:/data/monoliths --name wsm.rest -p 8081:80 ldesousa/wsm.rest
 
 # Declaring variables inline:
-# docker run -dit -e DB_HOST=scomp1270.wurnet.nl -e DB_NAME=isric  -e DB_USER=haas005  -e DB_PASSWORD=wur -e DB_PORT=5479 -v /data/monoliths:/data/monoliths --name wsm.rest -p 8081:80 ldesousa/wsm.rest
\ No newline at end of file
+# docker run -dit -e DB_HOST=scomp1270.wurnet.nl -e DB_NAME=isric  -e DB_USER=haas005  -e DB_PASSWORD=wur -e DB_PORT=5479 -v /data/monoliths:/data/monoliths --name wsm.rest -p 8081:80 ldesousa/wsm.rest

RESTServer.wsgi

-import sys,os
-
-#sys.path.append('../')
-sys.path.insert(0,os.path.abspath(os.path.dirname(__file__)))
-
-from appWSM import app as application

appWSM.py

 #https://github.com/humiaozuzu/awesome-flask
 from flask import Flask,make_response
 from flask_restful import Api
-from flask_cors import CORS
 
 from resources.meta import Monoliths,Monolith,MonolithRoot
 
 import logging,os.path
 
 app = Flask(__name__)
-CORS(app, resources={r"/*": {'origins': '*'}})
 api = Api(app)
 
 basePath=os.path.dirname(__file__)
 
-file_handler = logging.FileHandler(os.path.join(basePath,'app.log'))
-app.logger.addHandler(file_handler)
-app.logger.setLevel(logging.INFO)
-
+#file_handler = logging.FileHandler(os.path.join(basePath,'app.log'))
+#app.logger.addHandler(file_handler)
+#app.logger.setLevel(logging.INFO)
+app.config["LOG_TYPE"] = os.environ.get("LOG_TYPE", "stream")
+app.config["LOG_LEVEL"] = os.environ.get("LOG_LEVEL", "INFO")
 
  #'/monoliths/<path:path>',
 api.add_resource(Monolith,"/monoliths","/")

common/filters.py

@@ -38,8 +38,6 @@ def filter_extra(fileList):
     [{u'JM-006-def.1per.jpg': {'profile_id': 63764, 'publication_year': 2006, 'classification': 'Cambisols'}}, {u'FR-011-def.1per.jpg': {'profile_id': 69755, 'publication_year': 2006, 'classification': 'Albeluvisols'}}, {u'IT-016-def.1per.jpg': None}]
     """
     #connect to database
-    #conn = psycopg2.connect(host="85.214.35.184",database="isric", user="jjesus", password="isric126")
-    #dictCur = conn.cursor(cursor_factory=psycopg2.extras.DictCursor)
     dictCur=get_db_connection()
     #dictCur.execute("select * from rest.vw_isis_classification_profile_geom")
     #dictCur.execute("select * from web_isis.vw_isis_profiles_with_classification_geo_img_rest")

common/secret.py

@@ -3,10 +3,10 @@
 
 from os import environ
 # Get environment variables and set default if none specified:
-DB_HOST = environ.get("DB_HOST", "scomp1270.wurnet.nl")
-DB_NAME = environ.get("DB_NAME", "isric")
-DB_USER = environ.get("DB_USER", "haas005")
-DB_PASSWORD = environ.get("DB_PASSWORD", "1515NewPW")
-DB_PORT = environ.get("DB_PORT", "5479")
+DB_HOST = environ.get("DB_HOST")
+DB_NAME = environ.get("DB_NAME")
+DB_USER = environ.get("DB_USER")
+DB_PASSWORD = environ.get("DB_PASSWORD")
+DB_PORT = environ.get("DB_PORT")
 
 DB={"host":DB_HOST,"database":DB_NAME, "user":DB_USER, "password":DB_PASSWORD,"port":DB_PORT}

common/util.py

@@ -61,4 +61,4 @@ def get_url_params(testing=False,req=None):
 if __name__ == "__main__":
     #print get_url_params(testing=True,req=[('country', [u'BR,ES']), ('limit', [u'15.3'])])
     import doctest
-    doctest.testmod(verbose=1)
\ No newline at end of file
+    doctest.testmod(verbose=1)

requirements.txt

-aniso8601==1.3.0
-click==6.7
-Flask==0.12.2
-Flask-Cors==3.0.6
-Flask-RESTful==0.3.6
-flask-restplus==0.10.1
-itsdangerous==0.24
-Jinja2==2.10
-jsonschema==2.6.0
-MarkupSafe==1.0
-natsort==5.1.1
-psycopg2==2.7.3.2
-python-dateutil==2.6.1
-pytz==2017.3
-six==1.11.0
-Werkzeug==0.13
-gunicorn==19.7.1
\ No newline at end of file
+aniso8601==9.0.1
+attrs==21.2.0
+click==7.1.2
+eventlet==0.30.2
+Flask==1.1.4
+Flask-RESTful==0.3.9
+flask-restplus==0.13.0
+gunicorn==20.1.0
+itsdangerous==1.1.0
+Jinja2==2.11.3
+jsonschema==3.2.0
+MarkupSafe==2.0.1
+natsort==7.1.1
+psycopg2==2.9.1
+pyrsistent==0.17.3
+pytz==2021.1
+six==1.16.0
+Werkzeug==1.0.1

resources/meta.py

-#from flask_restful import Resource
 from flask import current_app
 from flask import render_template,make_response,send_file
+
+# ISSUE: https://github.com/noirbizarre/flask-restplus/issues/777
+import werkzeug 
+werkzeug.cached_property = werkzeug.utils.cached_property
+
 from flask_restplus import Resource
 import json
-import os.path,os
+import os,os.path
 from natsort import natsorted
 
 from common.util import get_url_params
 from common.filters import filter_countries,filter_limit,filter_extra,filter_classification
 
+from os import environ
+
 class Logger(object):
     """We cant get the logging object in the resources"""
     def info(self):
-         app = current_app._get_current_object()
-         return app.logger
+        app = current_app._get_current_object()
+        return app.logger
 
-
-     
 class Monolith(Resource):
     """Class that return the monoliths available"""
     def __init__(self):
         self.logger = Logger().info()
     def get(self):
-       #return 'You want path: %s' % path
-       headers = {'Content-Type': 'text/html'}
-       #return render_template('index.html')
-       return make_response(render_template('index.html'),200,headers)
-       #return "Yes I have monoliths"
+        headers = {'Content-Type': 'text/html'}
+        return make_response(render_template('index.html'),200,headers)
+
 
 class MonolithRoot(Resource):
     """Class that return the monoliths available"""
     def __init__(self):
         self.logger = Logger().info()
     def get(self):
-       #return 'You want path: %s' % path
-       #headers = {'Content-Type': 'text/html'}
-       #return render_template('index.html')
-       #return make_response(render_template('index.html'),200,headers)
-       headers = {'Content-Type': 'application/json'}
-       monolithsPath=os.path.join( os.path.dirname( __file__ ), '..',"./monoliths/" )
-       #listDir=[o for o in os.listdir("./monoliths/") if os.path.isdir("./monoliths/"+o)]
-       listDir=[o for o in os.listdir(monolithsPath) if os.path.isdir(os.path.join(monolithsPath,o))]
-       
+        headers = {'Content-Type': 'application/json'}
+        monolithsPath = environ.get("MONOLITH_PATH")
+        listDir=[o for o in os.listdir(monolithsPath) if os.path.isdir(os.path.join(monolithsPath,o))]
        
-       response = json.dumps(listDir)   
-       return make_response(response,200,headers)
+        response = json.dumps(listDir)   
+        return make_response(response,200,headers)
 
 class CloseBy(Resource):
     def __init__(self):
@@ -57,43 +53,37 @@ class CloseBy(Resource):
     
 class Monoliths(Resource):
     def __init__(self):
-       self.logger = Logger().info()
+        self.logger = Logger().info()
+    
     def get(self,path):
-       monolithPath=os.path.join( os.path.dirname( __file__ ), '..',"./monoliths/",path )
-       #monolithPath=os.path.join("./monoliths/"+path)
-       #self.logger.info("./monoliths/"+path)
-       #if os.path.isdir(relativePath)
+        monolithPath = os.path.join(environ.get("MONOLITH_PATH"),path)
+        #monolithPath=os.path.join("./monoliths/"+path)
+        #self.logger.info("./monoliths/"+path)
+        #if os.path.isdir(relativePath)
       
-       #http://127.0.0.1:5000/monoliths/1per/jpg?country=BR,ES
-       if os.path.isdir(monolithPath):
-           #send JSON with content
-           #http://127.0.0.1:5000/monoliths/10per/jpg
-           listFile=natsorted(os.listdir(monolithPath))
-
-   
-           req=get_url_params()
+        #http://127.0.0.1:5000/monoliths/1per/jpg?country=BR,ES
+        if os.path.isdir(monolithPath):
+            #send JSON with content
+            #http://127.0.0.1:5000/monoliths/10per/jpg
+            listFile = natsorted(os.listdir(monolithPath))
+            req=get_url_params()
             
-           if req["country"]:
-               listFile=filter_countries(req["country"],listFile)
+            if req["country"]:
+                listFile=filter_countries(req["country"],listFile)
                         
-           if req["classification"]:
-               listFile=filter_classification(req["classification"],listFile)
+            if req["classification"]:
+                listFile=filter_classification(req["classification"],listFile)
                
-           if req["limit"]:
-               listFile=filter_limit(req["limit"] ,listFile)
-           #PASOP THIS HAS TO BE THE LAST FILTER
-           if req["extra"]:
-               listFile=filter_extra(listFile)
-          
-               
-           headers = {'Content-Type': 'application/json'}
-           response = json.dumps(listFile)
-           return make_response(response,200,headers)
-          #return json.dumps(os.listdir(relativePath)) #testing URL http://127.0.0.1:5000/monoliths/10per/jpg
-       else:
-           #send file
-           return send_file(monolithPath)
-           #send_file 
-           #return 'You want path: %s' % str(os.path.isdir(relativePath))
-       #return 1
-   
\ No newline at end of file
+            if req["limit"]:
+                listFile=filter_limit(req["limit"] ,listFile)
+            #PASOP THIS HAS TO BE THE LAST FILTER
+            if req["extra"]:
+                listFile=filter_extra(listFile)
+
+            headers = {'Content-Type': 'application/json'}
+            response = json.dumps(listFile)
+            return make_response(response,200,headers)
+            #return json.dumps(os.listdir(relativePath)) #testing URL http://127.0.0.1:5000/monoliths/10per/jpg
+        else:
+            #send file
+            return send_file(monolithPath)

tools/createTableImage.py

+#select p.country_id,p.geom,p.profile_id,p.profile_code,cl.classification from web_isis.vw_isis_profile as p left join  web_isis.vw_isis_wrb_classification_profile as cl ON cl.profile_id=p.profile_id where (p.geom is NOT NULL and cl.classification<>'');    
+
+#creating table CREATE TABLE rest.images(country_id character(2) NOT NULL,profile_id integer NOT NULL,profile_code character varying(80) NOT NULL, reference_soil_group character varying(30) NOT NULL,file_path character varying(250) NOT NULL, geom geometry);
+
+import psycopg2
+import psycopg2.extras
+import glob, re,csv
+SQLClassTable = """select distinct p.country_id,p.geom,p.profile_id,p.profile_code,cl.classification from web_isis.vw_isis_profile as p left join  web_isis.vw_isis_wrb_classification_profile as cl ON cl.profile_id=p.profile_id where (p.geom is NOT NULL and cl.classification<>'')"""
+
+  
+
+
+tiffLocation="/mnt/ISRIC_Massive_Storage/Physical_collections/Photography/Photos-original/*.tif"
+fileList=glob.glob(tiffLocation)
+
+
+
+#/mnt/ISRIC_Massive_Storage/Physical_collections/Photography/Photos-original/*.tif
+
+conn = psycopg2.connect("dbname='isric' user='jjesus' host='85.214.35.184' password='isric126'")
+
+cur = conn.cursor(cursor_factory = psycopg2.extras.NamedTupleCursor)
+cur.execute(SQLClassTable)
+res = cur.fetchall()
+
+#Record(country_id='FI', geom='0101000020E61000004F1BE8B4812E38406DA0D3063AB54E40', profile_id=69734, profile_code='FI002', classification='Podzols')
+
+
+
+
+f1=open("/tmp/f1.csv","w")
+w=csv.writer(f1)
+w.writerow(("country_id","profile_id","profile_code","classification","path"))
+
+
+for record in res:
+    profile_code=record.profile_code[0:2]+'-'+record.profile_code[2:5]
+    fileOK = [x for x in fileList if re.search(profile_code, x)]
+    if fileOK:
+       w.writerow([record.country_id,record.profile_id,record.profile_code,record.classification,str(fileOK)])
+       
+           
+
+print "done"
+#try:
+#    conn = psycopg2.connect("dbname='web_isis' user='jjesus' host='85.214.35.184' password='isric126'")
+#except:
+#    print "I am unable to connect to the database"
+

tools/imageProcessing/GeomNoPics.csv

+#Missing pictures, profiles on ISIS with coordinates but no pictures
+AU-012
+AU-016
+AU-009
+AU-030
+AU-032
+AU-010
+AU-039
+AU-011
+AU-006
+AU-029
+AU-021
+AU-008
+AU-033
+AU-018
+AU-015
+BE-003
+BE-001
+BE-001
+BE-002
+BJ-001
+BR-013
+CA-017
+CA-014
+CD-002
+CD-001
+CI-006
+CI-007
+CM-001
+CN-016
+CN-015
+CN-011
+CN-010
+CN-050
+CN-051
+CN-048
+CN-049
+CN-018
+CN-006
+CN-005
+CN-007
+CN-032
+CN-029
+CN-025
+CN-022
+CN-024
+CN-028
+CN-026
+CN-001
+CN-035
+CN-042
+CN-037
+CO-014
+CO-018
+CR-003
+CU-004
+CU-016
+CU-015
+DE-004
+DK-003
+DK-001
+EC-020
+EC-018
+EC-001
+FI-005
+FI-003
+FR-007
+GL-002
+GL-004
+ID-023
+ID-013
+ID-025
+IS-007
+JM-001
+JM-002
+JM-003
+JM-004
+JM-007
+JM-005
+KE-018
+KE-019
+KE-026
+KE-025
+KE-035
+KE-037
+KE-002
+KE-016
+KE-005
+KE-034
+KE-023
+KE-024
+KE-038
+KE-007
+KE-022
+KE-039
+KE-042
+KE-041
+KE-043
+KE-021
+KE-046
+KE-045
+KE-036
+KE-047
+KE-050
+KE-008
+KE-055
+KE-014
+KE-012
+KE-033
+KE-031
+KE-063
+KE-032
+KE-029
+KE-011
+KE-030
+KE-028
+LK-001
+MZ-009
+MZ-001
+MZ-002
+NA-006
+NA-004
+NA-008
+NG-020
+NG-017
+NG-025
+NG-022
+NG-015
+NI-011
+NI-005
+NI-004
+NI-009
+NI-008
+NL-035
+NL-005
+NL-018
+NZ-003
+PE-012
+PE-015
+PE-013
+PE-014
+PE-017
+PE-018
+PE-011
+RU-004
+RU-010
+RU-009
+RU-011
+RU-012
+RU-013
+RU-002
+RU-015
+RU-014
+RW-002
+RW-001
+SK-001
+SK-002
+SK-006
+SK-005
+SK-004
+SK-003
+SK-007
+TJ-001
+US-005
+US-006
+US-007
+US-027
+UY-008
+ZM-003
+ZW-010
+ZW-008
+ZW-013

tools/imageProcessing/Original2ResizeImages.py

+"""
+Update: Ulan Turdukulov, 6th October 2020
+1) Moved code to PIL library (Pillow): pip install Pillow
+2) PIL doesn't have percent resize option - used width and height of the original image
+Note, some formats (jpg) require RGB conversion
+
+
+From a specific folder containing tiff original files will open the file and 10per, 
+20_200 1per 40_40050per, 80_800, original
+
+"""
+
+path_str = """ 
+├── 10per
+│   ├── jpg
+│   ├── png
+│   └── tif
+├── 1per
+│   ├── jpg
+│   ├── png
+│   └── tif
+├── 40_400
+│   ├── jpg
+│   ├── png
+│   └── tif
+└── original
+    ├── jpg
+    ├── png
+    └── tif
+"""
+import os
+import glob
+from PIL import Image
+
+
+# import shlex
+# import click
+# @click.command()
+# @click.option('--make_path', is_flag = True, default = True ,help = "Will generate outputs path on destination folde")
+# @click.option('--in_path',nargs=1, type=click.Path(exists=True), required=True, help = 'Path with original tiff images')
+# @click.option('--out_path', nargs=1, type=click.Path(exists=True), required=True,