Commit 94aec03b authored by Jorge S. Mendes de Jesus's avatar Jorge S. Mendes de Jesus
Browse files

Merge branch 'develop'

	common/secret.py
parents 462f64d4 bef7478d
monoliths
wsm.rest.env
wsm.rest.sock
tools
.git
.gitgnore
.project
.pydevproject
*/*.pyc
__pycache__
\ No newline at end of file
#VSCode and Eclipse
.vscode
*.code-workspace
.project
.pydevproject
# Local History for Visual Studio Code
.history/
# Local envs
.env
# Application Specific
monoliths
*.pyc
app.log
wsm.rest.env
wsm.rest.sock
logs
.project
.pydevproject
# Python
__pycache__
default:
interruptible: false
stages:
- tag
- publish
# tag release
tag:
image: node:12-buster-slim
stage: tag
before_script:
- apt-get update && apt-get install -y --no-install-recommends git-core ca-certificates
- npm install -g semantic-release @semantic-release/changelog @semantic-release/commit-analyzer @semantic-release/gitlab @semantic-release/git @semantic-release/npm @semantic-release/release-notes-generator conventional-changelog conventional-changelog-conventionalcommits
script:
- semantic-release
when: on_success
only:
- master
except:
refs:
- tags
variables:
- $CI_COMMIT_TITLE =~ /^RELEASE:.+$/
# build a new container
build:
stage: publish
image:
name: gcr.io/kaniko-project/executor:debug
entrypoint: [""]
when: on_success
script:
- mkdir -p /kaniko/.docker
- echo "{\"auths\":{\"$HARBOR_REGISTRY\":{\"username\":\"$HARBOR_USER\",\"password\":\"$HARBOR_PASSWORD\"}}}" > /kaniko/.docker/config.json
- if [ -z $flag ]; then /kaniko/executor --context $CI_PROJECT_DIR --dockerfile $CI_PROJECT_DIR/Dockerfile --destination $HARBOR_REGISTRY/$HARBOR_PROJECT/$CI_PROJECT_NAME:$CI_COMMIT_SHORT_SHA; else /kaniko/executor --context $CI_PROJECT_DIR --dockerfile $CI_PROJECT_DIR/Dockerfile --destination $HARBOR_REGISTRY/$HARBOR_PROJECT/$CI_PROJECT_NAME:$CI_COMMIT_TAG --destination $HARBOR_REGISTRY/$HARBOR_PROJECT/$CI_PROJECT_NAME:latest; fi
only:
- tags
- develop
<?xml version="1.0" encoding="UTF-8" standalone="no"?>
<?eclipse-pydev version="1.0"?><pydev_project>
<pydev_property name="org.python.pydev.PYTHON_PROJECT_INTERPRETER">Default</pydev_property>
<pydev_property name="org.python.pydev.PYTHON_PROJECT_VERSION">python interpreter</pydev_property>
</pydev_project>
plugins:
- "@semantic-release/gitlab"
branches: ['master']
ci: true
debug: true
dryRun: false
tagFormat: '${version}'
# Global plugin options (will be passed to all plugins)
preset: 'conventionalcommits'
gitlabUrl: 'http://git.wur.nl/' # your gitlab url
# slackWebhook: 'https://slack.xxx.com/hooks/q3dtkec6yjyg9x6616o3atgkkr' # if you need slack notifies
# Responsible for verifying conditions necessary to proceed with the release:
# configuration is correct, authentication token are valid, etc...
verifyConditions:
- '@semantic-release/changelog'
- '@semantic-release/git'
- '@semantic-release/gitlab'
# Responsible for determining the type of the next release (major, minor or patch).
# If multiple plugins with a analyzeCommits step are defined, the release type will be
# the highest one among plugins output.
# Look details at: https://github.com/semantic-release/commit-analyzer#configuration
analyzeCommits:
- path: '@semantic-release/commit-analyzer'
# Responsible for generating the content of the release note.
# If multiple plugins with a generateNotes step are defined,
# the release notes will be the result of the concatenation of each plugin output.
generateNotes:
- path: '@semantic-release/release-notes-generator'
writerOpts:
groupBy: 'type'
commitGroupsSort: 'title'
commitsSort: 'header'
linkCompare: true
linkReferences: true
# Responsible for preparing the release, for example creating or updating files
# such as package.json, CHANGELOG.md, documentation or compiled assets
# and pushing a commit.
prepare:
- path: '@semantic-release/changelog'
- path: '@semantic-release/git'
message: 'RELEASE: ${nextRelease.version}'
assets: ['CHANGELOG.md']
# Responsible for publishing the release.
publish:
- path: '@semantic-release/gitlab'
APP_MODULE=appWSM:app
FROM ubuntu:16.04
MAINTAINER Luís de Sousa <luis.desousa@wur.nl>
FROM python:3.9.5-slim-buster@sha256:076f9edf940d59446b98c242de0058054240b890a47d1dbed18560d12ec58228
RUN apt update
RUN apt -y install python-pip python-dev nginx
# Required volume for monoliths e.g:
# docker volume create monoliths-volume
# docker build -t wsm.rest
# docker run -it -v /data/monoliths-volume:/data/monoliths .
COPY ./requirements.txt /tmp/requirements.txt
RUN pip install -r /tmp/requirements.txt
LABEL "maintainer"="jorge.mendesdejesus@isric.org"
# Expose port 80
EXPOSE 80
ENV LANG=C.UTF-8 LC_ALL=C.UTF-8
ENV DEBIAN_FRONTEND noninteractive
################### Usage #####################
RUN apt-get update \
&& apt-get install -y --no-install-recommends postgresql-client build-essential libpq-dev tini
# Build it
# docker build -t ldesousa/wsm.rest .
RUN mkdir -p /app
# Mount point with data
RUN mkdir -p /data/monoliths
WORKDIR /app
# Run command
# docker run -dit -v /var/www/wsm.rest:/var/www/wsm.rest -v /data/monoliths:/data/monoliths --name wsm.rest -p 8081:80 ldesousa/wsm.rest
COPY . .
# Set the correct database connection in common/secret.py
RUN pip install -r requirements.txt
# Attach to the container and run the startup script
# docker attach wsm.rest
# sh /var/www/wsm.rest/configs/startup.sh
# Ctrl+P Ctrl+Q
# Clean up
RUN pip cache purge \
&& apt remove -y build-essential \
&& apt -y autoremove \
&& rm -rf /var/lib/apt/lists/*
USER www-data
EXPOSE 8000
ENTRYPOINT ["/usr/bin/tini","-g","--"]
CMD ["gunicorn","--bind","0.0.0.0","--workers","2","--threads","4","--worker-class","eventlet", "--log-file=-", "wsgi:app"]
WSM
---
# WSM
World Soil Museum - REST API
App structure
-------------
## App structure
REST APP is on [appWSM.py](./appWSM.py) and then calling modules like `resources`. This app is based on:
......@@ -25,18 +23,23 @@ App accesses ISRIC database on `scomp1270.wurnet.nl`.
**NOTE:** To copy files check section below [Monolith picture location](#monolith-picture-location)
### DB access configuration / Data path
### DB access configuration
DB options and monolith path location is picked up from env variables on script: [common/secret.py](./common/secret.py).
DB options are picked up from file [common/secret.py](./common/secret.py). This file needs to be changed if a new DB is used.
NOTE: Jorge did a bad job on leaving DB password on the repository. Password should be on envs.
```bash
DB_HOST=scomp1270.wurnet.nl
DB_NAME=isric
DB_USER=hass005
DB_PASSWORD=<removed>
DB_PORT=5479
MONOLITH_PATH=/data/monolith
```
### DB queries
In the code we have the following queries:
| code path | method/function | SQL |
|:-----------------:|:---------------------:|:-----------------------------:|
| common/filters.py | filter_classification()|select * from web_isis.vw_isis_profiles_with_classification_geo_img_rest where classification ilike %s"|
......@@ -44,40 +47,35 @@ NOTE: Jorge did a bad job on leaving DB password on the repository. Password sho
**NOTE:** Views need to be refreshed when new data is added to the upstream tables
Monolith picture location
-------------------------
### Monolith picture location
Monoliths used by REST are stored on ISRIC data volume, the access to this volume is very restricted and based on IP. Rserver (scomp1457.wurnet.nl), on this server users haas005, turku001 and ruipe001 have a symbolic link to the monoliths.
There the monolith pictures can be updated or modified
Monolith file copy
------------------
### Monolith file copy
On the Rserver is it better to copy the monoliths from workspace scratch into `ISRIC_Monoliths`. On the home folder of user the command would look like:
```
```bash
turdu001@scomp1457:~$ cp -rv ./ISRIC_Workspace/scratch/Upload_ToRest/combined_processed/* ./ISRIC_Monoliths/
```
The FB-IT mounts are not reliable, check if "ISRIC_Monolith" is accessible, it not mount it:
```
```bash
# mount all volumes from /etc/fstab
sudo mount -a
```
In case of batch file permission it is advisable (NOTE: Only sudo can do it):
```
```bash
find /mnt/ISRIC_Monoliths -type d -exec chmod 777 {} \; #for directories
find /mnt/ISRIC_Monoliths -type f -exec chmod 666 {} \; #for files
```
Monolith image update and implementation
----------------------------------------
## Monolith image update and implementation
To update monolith images, it is necessary:
......@@ -89,8 +87,7 @@ It is advisable to use a rsync software when moving files around.
REST API doesn't make any assumptions on files and will crawl the fold structure for the pictures. It maybe necessary some code changes if there are to many changes.
Local Deployment
----------------
## Local Deployment
This API can be deployed using a docker container. The code resides in the
host system, together with the monoliths images to be served. These images
......@@ -100,42 +97,45 @@ as this file). This code folder is then mapped into the container at startup.
The Dockerfile present in this project sets up the API with Gunicorn and nginx.
To build the container image change to the project folder and run:
`docker build -t ldesousa/wsm.rest .`
`docker build -t wsm.rest .`
The resulting image is named `ldesousa/wsm.rest`. To run the image it is
necessary to map the source code folder and port 80 (where nginx listens):
The resulting image is named `wsm/rest`.
`docker run -dit -v /var/www/wsm.rest:/var/www/wsm.rest --name wsm.rest -p 8081:80 ldesousa/wsm.rest`
To run the image it is necessary to map the right environment variables to connect to the database as well as mount the source code folder and bind port 8000 (where gunicorn listens):
Gunicorn can only be started after the source code folder has been mapped. To
do so attach to the running container (named `wsm.rest`):
Docker run command with envfile:
`docker attach wsm.rest`
And then run the startup script:
`sh /var/www/wsm.rest/configs/startup.sh`
Finally, detach orderly from the running container with the escape sequence:
`Ctrl+P Ctrl+Q`
```bash
docker run -dit \
--env-file /path/to/envfile \
-v /data/monoliths:/data/monoliths \
--name wsm.rest -p 8000:8000 wsm.rest
```
OC deployment
=============
Docker run command with variables inline:
```bash
docker run -dit \
-e DB_HOST=scomp1270.wurnet.nl \
-e DB_NAME=isric \
-e DB_USER=haas005 \
-e DB_PASSWORD=<REMOVED>
-e MONOLITH_PATH=/data/monolith
-e DB_PORT=5479 \
-v /data/monoliths:/data/monoliths
--name wsm.rest -p 8000:8000 wsm.rest
```
OC uses the gitlab webhook to deploy the application on project `isric-files`. Pod application is named `rest-wsm`, and external route is `rest-wsm.isric.org`.
Your container will now be available on [localhost:8000](http://localhost:8000)
Image build is based on the `python3.6` OC image `openshift/python:3.6` and WSGI system is configure with envs:
## K8s deployment
```
APP_MODULE=appWSM:app
```
- `ENV` variables are set as k8s secrets on the [ISRIC deployment repository](https://git.wur.nl/isric/ict/k8s-deployments)
Envs are stored on configuration file [here](.s2i/environment)
- Application follows the standard k8s deployment.
Pod mounts the `isric-files/monoliths` on `/opt/app-root/src/monoliths (ro)`
- TODO: Write about semanatic tagging
Image processing and metadata
=============================
## Image processing and metadata
Please consult the specific folder **[README](imageProcessing/README.md)**
import sys,os
#sys.path.append('../')
sys.path.insert(0,os.path.abspath(os.path.dirname(__file__)))
from appWSM import app as application
#https://github.com/humiaozuzu/awesome-flask
from flask import Flask,make_response
from flask_restful import Api
from flask_cors import CORS
from resources.meta import Monoliths,Monolith,MonolithRoot
import logging,os.path
app = Flask(__name__)
CORS(app, resources={r"/*": {'origins': '*'}})
api = Api(app)
basePath=os.path.dirname(__file__)
file_handler = logging.FileHandler(os.path.join(basePath,'app.log'))
app.logger.addHandler(file_handler)
app.logger.setLevel(logging.INFO)
#file_handler = logging.FileHandler(os.path.join(basePath,'app.log'))
#app.logger.addHandler(file_handler)
#app.logger.setLevel(logging.INFO)
app.config["LOG_TYPE"] = os.environ.get("LOG_TYPE", "stream")
app.config["LOG_LEVEL"] = os.environ.get("LOG_LEVEL", "INFO")
#'/monoliths/<path:path>',
api.add_resource(Monolith,"/monoliths","/")
......
# This file must be used with "source bin/activate" *from bash*
# you cannot run it directly
deactivate () {
unset -f pydoc >/dev/null 2>&1
# reset old environment variables
# ! [ -z ${VAR+_} ] returns true if VAR is declared at all
if ! [ -z "${_OLD_VIRTUAL_PATH+_}" ] ; then
PATH="$_OLD_VIRTUAL_PATH"
export PATH
unset _OLD_VIRTUAL_PATH
fi
if ! [ -z "${_OLD_VIRTUAL_PYTHONHOME+_}" ] ; then
PYTHONHOME="$_OLD_VIRTUAL_PYTHONHOME"
export PYTHONHOME
unset _OLD_VIRTUAL_PYTHONHOME
fi
# This should detect bash and zsh, which have a hash command that must
# be called to get it to forget past commands. Without forgetting
# past commands the $PATH changes we made may not be respected
if [ -n "${BASH-}" ] || [ -n "${ZSH_VERSION-}" ] ; then
hash -r 2>/dev/null
fi
if ! [ -z "${_OLD_VIRTUAL_PS1+_}" ] ; then
PS1="$_OLD_VIRTUAL_PS1"
export PS1
unset _OLD_VIRTUAL_PS1
fi
unset VIRTUAL_ENV
if [ ! "${1-}" = "nondestructive" ] ; then
# Self destruct!
unset -f deactivate
fi
}
# unset irrelevant variables
deactivate nondestructive
VIRTUAL_ENV="/home/duque004/git/WSM/rest/appWSM"
export VIRTUAL_ENV
_OLD_VIRTUAL_PATH="$PATH"
PATH="$VIRTUAL_ENV/bin:$PATH"
export PATH
# unset PYTHONHOME if set
if ! [ -z "${PYTHONHOME+_}" ] ; then
_OLD_VIRTUAL_PYTHONHOME="$PYTHONHOME"
unset PYTHONHOME
fi
if [ -z "${VIRTUAL_ENV_DISABLE_PROMPT-}" ] ; then
_OLD_VIRTUAL_PS1="$PS1"
if [ "x" != x ] ; then
PS1="$PS1"
else
PS1="(`basename \"$VIRTUAL_ENV\"`) $PS1"
fi
export PS1
fi
# Make sure to unalias pydoc if it's already there
alias pydoc 2>/dev/null >/dev/null && unalias pydoc
pydoc () {
python -m pydoc "$@"
}
# This should detect bash and zsh, which have a hash command that must
# be called to get it to forget past commands. Without forgetting
# past commands the $PATH changes we made may not be respected
if [ -n "${BASH-}" ] || [ -n "${ZSH_VERSION-}" ] ; then
hash -r 2>/dev/null
fi
# This file must be used with "source bin/activate.csh" *from csh*.
# You cannot run it directly.
# Created by Davide Di Blasi <davidedb@gmail.com>.
alias deactivate 'test $?_OLD_VIRTUAL_PATH != 0 && setenv PATH "$_OLD_VIRTUAL_PATH" && unset _OLD_VIRTUAL_PATH; rehash; test $?_OLD_VIRTUAL_PROMPT != 0 && set prompt="$_OLD_VIRTUAL_PROMPT" && unset _OLD_VIRTUAL_PROMPT; unsetenv VIRTUAL_ENV; test "\!:*" != "nondestructive" && unalias deactivate && unalias pydoc'
# Unset irrelevant variables.
deactivate nondestructive
setenv VIRTUAL_ENV "/home/duque004/git/WSM/rest/appWSM"
set _OLD_VIRTUAL_PATH="$PATH"
setenv PATH "$VIRTUAL_ENV/bin:$PATH"
if ("" != "") then
set env_name = ""
else
set env_name = `basename "$VIRTUAL_ENV"`
endif
# Could be in a non-interactive environment,
# in which case, $prompt is undefined and we wouldn't
# care about the prompt anyway.
if ( $?prompt ) then
set _OLD_VIRTUAL_PROMPT="$prompt"
set prompt = "[$env_name] $prompt"
endif
unset env_name
alias pydoc python -m pydoc
rehash
# This file must be used using `. bin/activate.fish` *within a running fish ( http://fishshell.com ) session*.
# Do not run it directly.
function deactivate -d 'Exit virtualenv mode and return to the normal environment.'
# reset old environment variables
if test -n "$_OLD_VIRTUAL_PATH"
set -gx PATH $_OLD_VIRTUAL_PATH
set -e _OLD_VIRTUAL_PATH
end
if test -n "$_OLD_VIRTUAL_PYTHONHOME"
set -gx PYTHONHOME $_OLD_VIRTUAL_PYTHONHOME
set -e _OLD_VIRTUAL_PYTHONHOME
end
if test -n "$_OLD_FISH_PROMPT_OVERRIDE"
# Set an empty local `$fish_function_path` to allow the removal of `fish_prompt` using `functions -e`.
set -l fish_function_path
# Erase virtualenv's `fish_prompt` and restore the original.
functions -e fish_prompt
functions -c _old_fish_prompt fish_prompt
functions -e _old_fish_prompt
set -e _OLD_FISH_PROMPT_OVERRIDE
end
set -e VIRTUAL_ENV
if test "$argv[1]" != 'nondestructive'
# Self-destruct!
functions -e pydoc
functions -e deactivate
end
end
# Unset irrelevant variables.
deactivate nondestructive
set -gx VIRTUAL_ENV "/home/duque004/git/WSM/rest/appWSM"
set -gx _OLD_VIRTUAL_PATH $PATH
set -gx PATH "$VIRTUAL_ENV/bin" $PATH
# Unset `$PYTHONHOME` if set.
if set -q PYTHONHOME
set -gx _OLD_VIRTUAL_PYTHONHOME $PYTHONHOME
set -e PYTHONHOME
end
function pydoc
python -m pydoc $argv
end
if test -z "$VIRTUAL_ENV_DISABLE_PROMPT"
# Copy the current `fish_prompt` function as `_old_fish_prompt`.
functions -c fish_prompt _old_fish_prompt
function fish_prompt
# Save the current $status, for fish_prompts that display it.
set -l old_status $status
# Prompt override provided?
# If not, just prepend the environment name.
if test -n ""
printf '%s%s' "" (set_color normal)
else
printf '%s(%s%s%s) ' (set_color normal) (set_color -o white) (basename "$VIRTUAL_ENV") (set_color normal)
end
# Restore the original $status
echo "exit $old_status" | source
_old_fish_prompt
end
set -gx _OLD_FISH_PROMPT_OVERRIDE "$VIRTUAL_ENV"
end
"""By using execfile(this_file, dict(__file__=this_file)) you will
activate this virtualenv environment.
This can be used when you must use an existing Python interpreter, not
the virtualenv bin/python
"""
try:
__file__
except NameError:
raise AssertionError(
"You must run this like execfile('path/to/activate_this.py', dict(__file__='path/to/activate_this.py'))")
import sys
import os
old_os_path = os.environ.get('PATH', '')
os.environ['PATH'] = os.path.dirname(os.path.abspath(__file__)) + os.pathsep + old_os_path
base = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
if sys.platform == 'win32':
site_packages = os.path.join(base, 'Lib', 'site-packages')
else:
site_packages = os.path.join(base, 'lib', 'python%s' % sys.version[:3], 'site-packages')
prev_sys_path = list(sys.path)
import site
site.addsitedir(site_packages)
sys.real_prefix = sys.prefix
sys.prefix = base
# Move the added items to the front of the path:
new_sys_path = []
for item in list(sys.path):
if item not in prev_sys_path:
new_sys_path.append(item)
sys.path.remove(item)
sys.path[:0] = new_sys_path
#!/home/duque004/git/WSM/rest/appWSM/bin/python2
# -*- coding: utf-8 -*-
import re
import sys
from setuptools.command.easy_install import main
if __name__ == '__main__':
sys.argv[0] = re.sub(r'(-script\.pyw|\.exe)?$', '', sys.argv[0])
sys.exit(main())
#!/home/duque004/git/WSM/rest/appWSM/bin/python2