Skip to content
GitLab
Commit 04088c8b authored by Jorge S. Mendes de Jesus's avatar Jorge S. Mendes de Jesus
Browse files

New k8s implementation

parent 9fed521d
Pipeline #24745 failed with stage
in 18 seconds
Hide whitespace changes
Inline Side-by-side
.s2i/environment deleted 100644 → 0
View file @ 9fed521d
APP_MODULE=appWSM:app
Dockerfile
View file @ 04088c8b
FROM ubuntu:18.04
LABEL MAINTAINER Luís de Sousa <luis.desousa@wur.nl>
FROM python:3.9.5-slim-buster@sha256:076f9edf940d59446b98c242de0058054240b890a47d1dbed18560d12ec58228
RUN apt update && apt -y install python-pip python-dev nginx curl && rm -rf /var/lib/apt/lists/*
RUN mkdir -p /var/www/wsm.rest
# Required volume for monoliths e.g:
# docker volume create monoliths-volume
# docker build -t wsm.rest
# docker run -it -v /data/monoliths-volume:/data/monoliths .
WORKDIR /var/www/wsm.rest
LABEL "maintainer"="jorge.mendesdejesus@isric.org"
ENV LANG=C.UTF-8 LC_ALL=C.UTF-8
ENV DEBIAN_FRONTEND noninteractive
ENV DB_HOST scomp1270.wurnet.nl
ENV DB_NAME isric
ENV DB_USER hass005
ENV DB_PASSWORD 1515NewPW
ENV DB_PORT 5479
ENV MONOLITH_PATH /data/monoliths
RUN apt-get update \
&& apt-get install -y --no-install-recommends postgresql-client build-essential libpq-dev tini
RUN mkdir -p /app
# Mount point with data
RUN mkdir -p /data/monoliths
WORKDIR /app
COPY . .
RUN chmod +x configs/startup.sh
RUN pip install -r requirements.txt
ENTRYPOINT [ "configs/startup.sh" ]
USER www-data
EXPOSE 8000
ENTRYPOINT ["/usr/bin/tini","-g","--"]
CMD ["gunicorn","--bind","0.0.0.0","--workers","2","--threads","4","--worker-class","eventlet", "--log-file=-", "wsgi:app"]
# build-essential \
#RUN apt-get update \
# && apt-get install -y --no-install-recommends \
# postgresql-client \
# build-essential \
# gdal-bin libgdal-dev libtbb2 tini
#WORKDIR /src
#RUN mkdir -p /soils
#COPY soils /soils/soils
#COPY requirements.txt /soils/requirements.txt
#COPY gunicorn.py /soils/gunicorn.py
#WORKDIR /soils
#RUN pip install -r requirements.txt
# Clean up
#RUN pip cache purge \
# && apt remove -y build-essential \
# && apt -y autoremove \
# && rm -rf /var/lib/apt/lists/*
#USER www-data
#EXPOSE 8000
#ENTRYPOINT ["/usr/bin/tini","-g","--"]
#CMD ["gunicorn","--config=./gunicorn.py", "soils:app"]
#FROM ubuntu:18.04
#LABEL MAINTAINER Luís de Sousa <luis.desousa@wur.nl>
#RUN apt update && apt -y install python-pip python-dev nginx curl && rm -rf /var/lib/apt/lists/*
#RUN mkdir -p /var/www/wsm.rest
#WORKDIR /var/www/wsm.rest
#COPY . .
#RUN chmod +x configs/startup.sh
#RUN pip install -r requirements.txt
#ENTRYPOINT [ "configs/startup.sh" ]
EXPOSE 80
STOPSIGNAL SIGQUIT
#EXPOSE 80
#STOPSIGNAL SIGQUIT
HEALTHCHECK CMD curl --fail http://localhost:80 || exit 1
CMD ["nginx", "-g", "daemon off;"]
#HEALTHCHECK CMD curl --fail http://localhost:80 || exit 1
#CMD ["nginx", "-g", "daemon off;"]
################### Usage #####################
# Build it
......@@ -29,4 +97,4 @@ CMD ["nginx", "-g", "daemon off;"]
# docker run -dit --env-file /path/to/envfile -v /data/monoliths:/data/monoliths --name wsm.rest -p 8081:80 ldesousa/wsm.rest
# Declaring variables inline:
# docker run -dit -e DB_HOST=scomp1270.wurnet.nl -e DB_NAME=isric -e DB_USER=haas005 -e DB_PASSWORD=wur -e DB_PORT=5479 -v /data/monoliths:/data/monoliths --name wsm.rest -p 8081:80 ldesousa/wsm.rest
\ No newline at end of file
# docker run -dit -e DB_HOST=scomp1270.wurnet.nl -e DB_NAME=isric -e DB_USER=haas005 -e DB_PASSWORD=wur -e DB_PORT=5479 -v /data/monoliths:/data/monoliths --name wsm.rest -p 8081:80 ldesousa/wsm.rest
RESTServer.wsgi deleted 100644 → 0
View file @ 9fed521d
import sys,os
#sys.path.append('../')
sys.path.insert(0,os.path.abspath(os.path.dirname(__file__)))
from appWSM import app as application
appWSM.py
View file @ 04088c8b
#https://github.com/humiaozuzu/awesome-flask
from flask import Flask,make_response
from flask_restful import Api
from flask_cors import CORS
from resources.meta import Monoliths,Monolith,MonolithRoot
import logging,os.path
app = Flask(__name__)
CORS(app, resources={r"/*": {'origins': '*'}})
api = Api(app)
basePath=os.path.dirname(__file__)
file_handler = logging.FileHandler(os.path.join(basePath,'app.log'))
app.logger.addHandler(file_handler)
app.logger.setLevel(logging.INFO)
#file_handler = logging.FileHandler(os.path.join(basePath,'app.log'))
#app.logger.addHandler(file_handler)
#app.logger.setLevel(logging.INFO)
app.config["LOG_TYPE"] = os.environ.get("LOG_TYPE", "stream")
app.config["LOG_LEVEL"] = os.environ.get("LOG_LEVEL", "INFO")
#'/monoliths/<path:path>',
api.add_resource(Monolith,"/monoliths","/")
......
common/filters.py
View file @ 04088c8b
......@@ -38,8 +38,6 @@ def filter_extra(fileList):
[{u'JM-006-def.1per.jpg': {'profile_id': 63764, 'publication_year': 2006, 'classification': 'Cambisols'}}, {u'FR-011-def.1per.jpg': {'profile_id': 69755, 'publication_year': 2006, 'classification': 'Albeluvisols'}}, {u'IT-016-def.1per.jpg': None}]
"""
#connect to database
#conn = psycopg2.connect(host="85.214.35.184",database="isric", user="jjesus", password="isric126")
#dictCur = conn.cursor(cursor_factory=psycopg2.extras.DictCursor)
dictCur=get_db_connection()
#dictCur.execute("select * from rest.vw_isis_classification_profile_geom")
#dictCur.execute("select * from web_isis.vw_isis_profiles_with_classification_geo_img_rest")
......
common/secret.py
View file @ 04088c8b
......@@ -3,10 +3,10 @@
from os import environ
# Get environment variables and set default if none specified:
DB_HOST = environ.get("DB_HOST", "scomp1270.wurnet.nl")
DB_NAME = environ.get("DB_NAME", "isric")
DB_USER = environ.get("DB_USER", "haas005")
DB_PASSWORD = environ.get("DB_PASSWORD", "1515NewPW")
DB_PORT = environ.get("DB_PORT", "5479")
DB_HOST = environ.get("DB_HOST")
DB_NAME = environ.get("DB_NAME")
DB_USER = environ.get("DB_USER")
DB_PASSWORD = environ.get("DB_PASSWORD")
DB_PORT = environ.get("DB_PORT")
DB={"host":DB_HOST,"database":DB_NAME, "user":DB_USER, "password":DB_PASSWORD,"port":DB_PORT}
common/util.py
View file @ 04088c8b
......@@ -61,4 +61,4 @@ def get_url_params(testing=False,req=None):
if __name__ == "__main__":
#print get_url_params(testing=True,req=[('country', [u'BR,ES']), ('limit', [u'15.3'])])
import doctest
doctest.testmod(verbose=1)
\ No newline at end of file
doctest.testmod(verbose=1)
requirements.txt
View file @ 04088c8b
aniso8601==1.3.0
click==6.7
Flask==0.12.2
Flask-Cors==3.0.6
Flask-RESTful==0.3.6
flask-restplus==0.10.1
itsdangerous==0.24
Jinja2==2.10
jsonschema==2.6.0
MarkupSafe==1.0
natsort==5.1.1
psycopg2==2.7.3.2
python-dateutil==2.6.1
pytz==2017.3
six==1.11.0
Werkzeug==0.13
gunicorn==19.7.1
\ No newline at end of file
aniso8601==9.0.1
attrs==21.2.0
click==7.1.2
eventlet==0.30.2
Flask==1.1.4
Flask-RESTful==0.3.9
flask-restplus==0.13.0
gunicorn==20.1.0
itsdangerous==1.1.0
Jinja2==2.11.3
jsonschema==3.2.0
MarkupSafe==2.0.1
natsort==7.1.1
psycopg2==2.9.1
pyrsistent==0.17.3
pytz==2021.1
six==1.16.0
Werkzeug==1.0.1
resources/meta.py
View file @ 04088c8b
#from flask_restful import Resource
from flask import current_app
from flask import render_template,make_response,send_file
# ISSUE: https://github.com/noirbizarre/flask-restplus/issues/777
import werkzeug
werkzeug.cached_property = werkzeug.utils.cached_property
from flask_restplus import Resource
import json
import os.path,os
import os,os.path
from natsort import natsorted
from common.util import get_url_params
from common.filters import filter_countries,filter_limit,filter_extra,filter_classification
from os import environ
class Logger(object):
"""We cant get the logging object in the resources"""
def info(self):
app = current_app._get_current_object()
return app.logger
app = current_app._get_current_object()
return app.logger
class Monolith(Resource):
"""Class that return the monoliths available"""
def __init__(self):
self.logger = Logger().info()
def get(self):
#return 'You want path: %s' % path
headers = {'Content-Type': 'text/html'}
#return render_template('index.html')
return make_response(render_template('index.html'),200,headers)
#return "Yes I have monoliths"
headers = {'Content-Type': 'text/html'}
return make_response(render_template('index.html'),200,headers)
class MonolithRoot(Resource):
"""Class that return the monoliths available"""
def __init__(self):
self.logger = Logger().info()
def get(self):
#return 'You want path: %s' % path
#headers = {'Content-Type': 'text/html'}
#return render_template('index.html')
#return make_response(render_template('index.html'),200,headers)
headers = {'Content-Type': 'application/json'}
monolithsPath=os.path.join( os.path.dirname( __file__ ), '..',"./monoliths/" )
#listDir=[o for o in os.listdir("./monoliths/") if os.path.isdir("./monoliths/"+o)]
listDir=[o for o in os.listdir(monolithsPath) if os.path.isdir(os.path.join(monolithsPath,o))]
headers = {'Content-Type': 'application/json'}
monolithsPath = environ.get("MONOLITH_PATH")
listDir=[o for o in os.listdir(monolithsPath) if os.path.isdir(os.path.join(monolithsPath,o))]
response = json.dumps(listDir)
return make_response(response,200,headers)
response = json.dumps(listDir)
return make_response(response,200,headers)
class CloseBy(Resource):
def __init__(self):
......@@ -57,43 +53,37 @@ class CloseBy(Resource):
class Monoliths(Resource):
def __init__(self):
self.logger = Logger().info()
self.logger = Logger().info()
def get(self,path):
monolithPath=os.path.join( os.path.dirname( __file__ ), '..',"./monoliths/",path )
#monolithPath=os.path.join("./monoliths/"+path)
#self.logger.info("./monoliths/"+path)
#if os.path.isdir(relativePath)
monolithPath = os.path.join(environ.get("MONOLITH_PATH"),path)
#monolithPath=os.path.join("./monoliths/"+path)
#self.logger.info("./monoliths/"+path)
#if os.path.isdir(relativePath)
#http://127.0.0.1:5000/monoliths/1per/jpg?country=BR,ES
if os.path.isdir(monolithPath):
#send JSON with content
#http://127.0.0.1:5000/monoliths/10per/jpg
listFile=natsorted(os.listdir(monolithPath))
req=get_url_params()
#http://127.0.0.1:5000/monoliths/1per/jpg?country=BR,ES
if os.path.isdir(monolithPath):
#send JSON with content
#http://127.0.0.1:5000/monoliths/10per/jpg
listFile = natsorted(os.listdir(monolithPath))
req=get_url_params()
if req["country"]:
listFile=filter_countries(req["country"],listFile)
if req["country"]:
listFile=filter_countries(req["country"],listFile)
if req["classification"]:
listFile=filter_classification(req["classification"],listFile)
if req["classification"]:
listFile=filter_classification(req["classification"],listFile)
if req["limit"]:
listFile=filter_limit(req["limit"] ,listFile)
#PASOP THIS HAS TO BE THE LAST FILTER
if req["extra"]:
listFile=filter_extra(listFile)
headers = {'Content-Type': 'application/json'}
response = json.dumps(listFile)
return make_response(response,200,headers)
#return json.dumps(os.listdir(relativePath)) #testing URL http://127.0.0.1:5000/monoliths/10per/jpg
else:
#send file
return send_file(monolithPath)
#send_file
#return 'You want path: %s' % str(os.path.isdir(relativePath))
#return 1
\ No newline at end of file
if req["limit"]:
listFile=filter_limit(req["limit"] ,listFile)
#PASOP THIS HAS TO BE THE LAST FILTER
if req["extra"]:
listFile=filter_extra(listFile)
headers = {'Content-Type': 'application/json'}
response = json.dumps(listFile)
return make_response(response,200,headers)
#return json.dumps(os.listdir(relativePath)) #testing URL http://127.0.0.1:5000/monoliths/10per/jpg
else:
#send file
return send_file(monolithPath)
tools/createTableImage.py 0 → 100644
View file @ 04088c8b
#select p.country_id,p.geom,p.profile_id,p.profile_code,cl.classification from web_isis.vw_isis_profile as p left join web_isis.vw_isis_wrb_classification_profile as cl ON cl.profile_id=p.profile_id where (p.geom is NOT NULL and cl.classification<>'');
#creating table CREATE TABLE rest.images(country_id character(2) NOT NULL,profile_id integer NOT NULL,profile_code character varying(80) NOT NULL, reference_soil_group character varying(30) NOT NULL,file_path character varying(250) NOT NULL, geom geometry);
import psycopg2
import psycopg2.extras
import glob, re,csv
SQLClassTable = """select distinct p.country_id,p.geom,p.profile_id,p.profile_code,cl.classification from web_isis.vw_isis_profile as p left join web_isis.vw_isis_wrb_classification_profile as cl ON cl.profile_id=p.profile_id where (p.geom is NOT NULL and cl.classification<>'')"""
tiffLocation="/mnt/ISRIC_Massive_Storage/Physical_collections/Photography/Photos-original/*.tif"
fileList=glob.glob(tiffLocation)
#/mnt/ISRIC_Massive_Storage/Physical_collections/Photography/Photos-original/*.tif
conn = psycopg2.connect("dbname='isric' user='jjesus' host='85.214.35.184' password='isric126'")
cur = conn.cursor(cursor_factory = psycopg2.extras.NamedTupleCursor)
cur.execute(SQLClassTable)
res = cur.fetchall()
#Record(country_id='FI', geom='0101000020E61000004F1BE8B4812E38406DA0D3063AB54E40', profile_id=69734, profile_code='FI002', classification='Podzols')
f1=open("/tmp/f1.csv","w")
w=csv.writer(f1)
w.writerow(("country_id","profile_id","profile_code","classification","path"))
for record in res:
profile_code=record.profile_code[0:2]+'-'+record.profile_code[2:5]
fileOK = [x for x in fileList if re.search(profile_code, x)]
if fileOK:
w.writerow([record.country_id,record.profile_id,record.profile_code,record.classification,str(fileOK)])
print "done"
#try:
# conn = psycopg2.connect("dbname='web_isis' user='jjesus' host='85.214.35.184' password='isric126'")
#except:
# print "I am unable to connect to the database"
tools/imageProcessing/GeomNoPics.csv 0 → 100644
View file @ 04088c8b
#Missing pictures, profiles on ISIS with coordinates but no pictures
AU-012
AU-016
AU-009
AU-030
AU-032
AU-010
AU-039
AU-011
AU-006
AU-029
AU-021
AU-008
AU-033
AU-018
AU-015
BE-003
BE-001
BE-001
BE-002
BJ-001
BR-013
CA-017
CA-014
CD-002
CD-001
CI-006
CI-007
CM-001
CN-016
CN-015
CN-011
CN-010
CN-050
CN-051
CN-048
CN-049
CN-018
CN-006
CN-005
CN-007
CN-032
CN-029
CN-025
CN-022
CN-024
CN-028
CN-026
CN-001
CN-035
CN-042
CN-037
CO-014
CO-018
CR-003
CU-004
CU-016
CU-015
DE-004
DK-003
DK-001
EC-020
EC-018
EC-001
FI-005
FI-003
FR-007
GL-002
GL-004
ID-023
ID-013
ID-025
IS-007
JM-001
JM-002
JM-003
JM-004
JM-007
JM-005
KE-018
KE-019
KE-026
KE-025
KE-035
KE-037
KE-002
KE-016
KE-005
KE-034
KE-023
KE-024
KE-038
KE-007
KE-022
KE-039
KE-042
KE-041
KE-043
KE-021
KE-046
KE-045
KE-036
KE-047
KE-050
KE-008
KE-055
KE-014
KE-012
KE-033
KE-031
KE-063
KE-032
KE-029
KE-011
KE-030
KE-028
LK-001
MZ-009
MZ-001
MZ-002
NA-006
NA-004
NA-008
NG-020
NG-017
NG-025
NG-022
NG-015
NI-011
NI-005
NI-004
NI-009
NI-008
NL-035
NL-005
NL-018
NZ-003
PE-012
PE-015
PE-013
PE-014
PE-017
PE-018
PE-011
RU-004
RU-010
RU-009
RU-011
RU-012
RU-013
RU-002
RU-015
RU-014
RW-002
RW-001
SK-001
SK-002
SK-006
SK-005
SK-004
SK-003
SK-007
TJ-001
US-005
US-006
US-007
US-027
UY-008
ZM-003
ZW-010
ZW-008
ZW-013
tools/imageProcessing/Original2ResizeImages.py 0 → 100644
View file @ 04088c8b
"""
Update: Ulan Turdukulov, 6th October 2020
1) Moved code to PIL library (Pillow): pip install Pillow
2) PIL doesn't have percent resize option - used width and height of the original image
Note, some formats (jpg) require RGB conversion
From a specific folder containing tiff original files will open the file and 10per,
20_200 1per 40_40050per, 80_800, original
"""
path_str = """
├── 10per
│   ├── jpg
│   ├── png
│   └── tif
├── 1per
│   ├── jpg
│   ├── png
│   └── tif
├── 40_400
│   ├── jpg
│   ├── png
│   └── tif
└── original
├── jpg
├── png
└── tif
"""
import os
import glob
from PIL import Image
# import shlex
# import click
# @click.command()
# @click.option('--make_path', is_flag = True, default = True ,help = "Will generate outputs path on destination folde")
# @click.option('--in_path',nargs=1, type=click.Path(exists=True), required=True, help = 'Path with original tiff images')
# @click.option('--out_path', nargs=1, type=click.Path(exists=True), required=True,
# help = f'Output path for transfomed images path structure: {path_str}')
# def main(make_path, in_path, out_path):