Commit 94aec03b authored by Jorge S. Mendes de Jesus's avatar Jorge S. Mendes de Jesus
Browse files

Merge branch 'develop'

	common/secret.py
parents 462f64d4 bef7478d
monoliths monoliths
wsm.rest.env tools
wsm.rest.sock
.git .git
.gitgnore .gitgnore
.project .project
.pydevproject .pydevproject
*/*.pyc */*.pyc
__pycache__
\ No newline at end of file
#VSCode and Eclipse
.vscode
*.code-workspace
.project
.pydevproject
# Local History for Visual Studio Code
.history/
# Local envs
.env
# Application Specific
monoliths monoliths
*.pyc *.pyc
app.log app.log
wsm.rest.env wsm.rest.env
wsm.rest.sock wsm.rest.sock
logs logs
.project
.pydevproject # Python
__pycache__
default:
interruptible: false
stages:
- tag
- publish
# tag release
tag:
image: node:12-buster-slim
stage: tag
before_script:
- apt-get update && apt-get install -y --no-install-recommends git-core ca-certificates
- npm install -g semantic-release @semantic-release/changelog @semantic-release/commit-analyzer @semantic-release/gitlab @semantic-release/git @semantic-release/npm @semantic-release/release-notes-generator conventional-changelog conventional-changelog-conventionalcommits
script:
- semantic-release
when: on_success
only:
- master
except:
refs:
- tags
variables:
- $CI_COMMIT_TITLE =~ /^RELEASE:.+$/
# build a new container
build:
stage: publish
image:
name: gcr.io/kaniko-project/executor:debug
entrypoint: [""]
when: on_success
script:
- mkdir -p /kaniko/.docker
- echo "{\"auths\":{\"$HARBOR_REGISTRY\":{\"username\":\"$HARBOR_USER\",\"password\":\"$HARBOR_PASSWORD\"}}}" > /kaniko/.docker/config.json
- if [ -z $flag ]; then /kaniko/executor --context $CI_PROJECT_DIR --dockerfile $CI_PROJECT_DIR/Dockerfile --destination $HARBOR_REGISTRY/$HARBOR_PROJECT/$CI_PROJECT_NAME:$CI_COMMIT_SHORT_SHA; else /kaniko/executor --context $CI_PROJECT_DIR --dockerfile $CI_PROJECT_DIR/Dockerfile --destination $HARBOR_REGISTRY/$HARBOR_PROJECT/$CI_PROJECT_NAME:$CI_COMMIT_TAG --destination $HARBOR_REGISTRY/$HARBOR_PROJECT/$CI_PROJECT_NAME:latest; fi
only:
- tags
- develop
<?xml version="1.0" encoding="UTF-8" standalone="no"?>
<?eclipse-pydev version="1.0"?><pydev_project>
<pydev_property name="org.python.pydev.PYTHON_PROJECT_INTERPRETER">Default</pydev_property>
<pydev_property name="org.python.pydev.PYTHON_PROJECT_VERSION">python interpreter</pydev_property>
</pydev_project>
plugins:
- "@semantic-release/gitlab"
branches: ['master']
ci: true
debug: true
dryRun: false
tagFormat: '${version}'
# Global plugin options (will be passed to all plugins)
preset: 'conventionalcommits'
gitlabUrl: 'http://git.wur.nl/' # your gitlab url
# slackWebhook: 'https://slack.xxx.com/hooks/q3dtkec6yjyg9x6616o3atgkkr' # if you need slack notifies
# Responsible for verifying conditions necessary to proceed with the release:
# configuration is correct, authentication token are valid, etc...
verifyConditions:
- '@semantic-release/changelog'
- '@semantic-release/git'
- '@semantic-release/gitlab'
# Responsible for determining the type of the next release (major, minor or patch).
# If multiple plugins with a analyzeCommits step are defined, the release type will be
# the highest one among plugins output.
# Look details at: https://github.com/semantic-release/commit-analyzer#configuration
analyzeCommits:
- path: '@semantic-release/commit-analyzer'
# Responsible for generating the content of the release note.
# If multiple plugins with a generateNotes step are defined,
# the release notes will be the result of the concatenation of each plugin output.
generateNotes:
- path: '@semantic-release/release-notes-generator'
writerOpts:
groupBy: 'type'
commitGroupsSort: 'title'
commitsSort: 'header'
linkCompare: true
linkReferences: true
# Responsible for preparing the release, for example creating or updating files
# such as package.json, CHANGELOG.md, documentation or compiled assets
# and pushing a commit.
prepare:
- path: '@semantic-release/changelog'
- path: '@semantic-release/git'
message: 'RELEASE: ${nextRelease.version}'
assets: ['CHANGELOG.md']
# Responsible for publishing the release.
publish:
- path: '@semantic-release/gitlab'
APP_MODULE=appWSM:app
FROM ubuntu:16.04 FROM python:3.9.5-slim-buster@sha256:076f9edf940d59446b98c242de0058054240b890a47d1dbed18560d12ec58228
MAINTAINER Luís de Sousa <luis.desousa@wur.nl>
RUN apt update # Required volume for monoliths e.g:
RUN apt -y install python-pip python-dev nginx # docker volume create monoliths-volume
# docker build -t wsm.rest
# docker run -it -v /data/monoliths-volume:/data/monoliths .
COPY ./requirements.txt /tmp/requirements.txt LABEL "maintainer"="jorge.mendesdejesus@isric.org"
RUN pip install -r /tmp/requirements.txt
# Expose port 80 ENV LANG=C.UTF-8 LC_ALL=C.UTF-8
EXPOSE 80 ENV DEBIAN_FRONTEND noninteractive
################### Usage ##################### RUN apt-get update \
&& apt-get install -y --no-install-recommends postgresql-client build-essential libpq-dev tini
# Build it RUN mkdir -p /app
# docker build -t ldesousa/wsm.rest . # Mount point with data
RUN mkdir -p /data/monoliths
WORKDIR /app
# Run command COPY . .
# docker run -dit -v /var/www/wsm.rest:/var/www/wsm.rest -v /data/monoliths:/data/monoliths --name wsm.rest -p 8081:80 ldesousa/wsm.rest
# Set the correct database connection in common/secret.py RUN pip install -r requirements.txt
# Attach to the container and run the startup script # Clean up
# docker attach wsm.rest RUN pip cache purge \
# sh /var/www/wsm.rest/configs/startup.sh && apt remove -y build-essential \
# Ctrl+P Ctrl+Q && apt -y autoremove \
&& rm -rf /var/lib/apt/lists/*
USER www-data
EXPOSE 8000
ENTRYPOINT ["/usr/bin/tini","-g","--"]
CMD ["gunicorn","--bind","0.0.0.0","--workers","2","--threads","4","--worker-class","eventlet", "--log-file=-", "wsgi:app"]
WSM # WSM
---
World Soil Museum - REST API World Soil Museum - REST API
App structure ## App structure
-------------
REST APP is on [appWSM.py](./appWSM.py) and then calling modules like `resources`. This app is based on: REST APP is on [appWSM.py](./appWSM.py) and then calling modules like `resources`. This app is based on:
...@@ -25,17 +23,22 @@ App accesses ISRIC database on `scomp1270.wurnet.nl`. ...@@ -25,17 +23,22 @@ App accesses ISRIC database on `scomp1270.wurnet.nl`.
**NOTE:** To copy files check section below [Monolith picture location](#monolith-picture-location) **NOTE:** To copy files check section below [Monolith picture location](#monolith-picture-location)
### DB access configuration / Data path
### DB access configuration DB options and monolith path location is picked up from env variables on script: [common/secret.py](./common/secret.py).
DB options are picked up from file [common/secret.py](./common/secret.py). This file needs to be changed if a new DB is used. ```bash
DB_HOST=scomp1270.wurnet.nl
NOTE: Jorge did a bad job on leaving DB password on the repository. Password should be on envs. DB_NAME=isric
DB_USER=hass005
DB_PASSWORD=<removed>
DB_PORT=5479
MONOLITH_PATH=/data/monolith
```
### DB queries ### DB queries
In the code we have the following queries: In the code we have the following queries:
| code path | method/function | SQL | | code path | method/function | SQL |
|:-----------------:|:---------------------:|:-----------------------------:| |:-----------------:|:---------------------:|:-----------------------------:|
...@@ -44,40 +47,35 @@ NOTE: Jorge did a bad job on leaving DB password on the repository. Password sho ...@@ -44,40 +47,35 @@ NOTE: Jorge did a bad job on leaving DB password on the repository. Password sho
**NOTE:** Views need to be refreshed when new data is added to the upstream tables **NOTE:** Views need to be refreshed when new data is added to the upstream tables
Monolith picture location ### Monolith picture location
-------------------------
Monoliths used by REST are stored on ISRIC data volume, the access to this volume is very restricted and based on IP. Rserver (scomp1457.wurnet.nl), on this server users haas005, turku001 and ruipe001 have a symbolic link to the monoliths. Monoliths used by REST are stored on ISRIC data volume, the access to this volume is very restricted and based on IP. Rserver (scomp1457.wurnet.nl), on this server users haas005, turku001 and ruipe001 have a symbolic link to the monoliths.
There the monolith pictures can be updated or modified There the monolith pictures can be updated or modified
Monolith file copy ### Monolith file copy
------------------
On the Rserver is it better to copy the monoliths from workspace scratch into `ISRIC_Monoliths`. On the home folder of user the command would look like: On the Rserver is it better to copy the monoliths from workspace scratch into `ISRIC_Monoliths`. On the home folder of user the command would look like:
``` ```bash
turdu001@scomp1457:~$ cp -rv ./ISRIC_Workspace/scratch/Upload_ToRest/combined_processed/* ./ISRIC_Monoliths/ turdu001@scomp1457:~$ cp -rv ./ISRIC_Workspace/scratch/Upload_ToRest/combined_processed/* ./ISRIC_Monoliths/
``` ```
The FB-IT mounts are not reliable, check if "ISRIC_Monolith" is accessible, it not mount it: The FB-IT mounts are not reliable, check if "ISRIC_Monolith" is accessible, it not mount it:
``` ```bash
# mount all volumes from /etc/fstab # mount all volumes from /etc/fstab
sudo mount -a sudo mount -a
``` ```
In case of batch file permission it is advisable (NOTE: Only sudo can do it): In case of batch file permission it is advisable (NOTE: Only sudo can do it):
``` ```bash
find /mnt/ISRIC_Monoliths -type d -exec chmod 777 {} \; #for directories find /mnt/ISRIC_Monoliths -type d -exec chmod 777 {} \; #for directories
find /mnt/ISRIC_Monoliths -type f -exec chmod 666 {} \; #for files find /mnt/ISRIC_Monoliths -type f -exec chmod 666 {} \; #for files
``` ```
## Monolith image update and implementation
Monolith image update and implementation
----------------------------------------
To update monolith images, it is necessary: To update monolith images, it is necessary:
...@@ -89,53 +87,55 @@ It is advisable to use a rsync software when moving files around. ...@@ -89,53 +87,55 @@ It is advisable to use a rsync software when moving files around.
REST API doesn't make any assumptions on files and will crawl the fold structure for the pictures. It maybe necessary some code changes if there are to many changes. REST API doesn't make any assumptions on files and will crawl the fold structure for the pictures. It maybe necessary some code changes if there are to many changes.
Local Deployment ## Local Deployment
----------------
This API can be deployed using a docker container. The code resides in the This API can be deployed using a docker container. The code resides in the
host system, together with the monoliths images to be served. These images host system, together with the monoliths images to be served. These images
must be accessible in a folder named `monoliths` at the folder root (same level must be accessible in a folder named `monoliths` at the folder root (same level
as this file). This code folder is then mapped into the container at startup. as this file). This code folder is then mapped into the container at startup.
The Dockerfile present in this project sets up the API with Gunicorn and nginx. The Dockerfile present in this project sets up the API with Gunicorn and nginx.
To build the container image change to the project folder and run: To build the container image change to the project folder and run:
`docker build -t ldesousa/wsm.rest .` `docker build -t wsm.rest .`
The resulting image is named `ldesousa/wsm.rest`. To run the image it is The resulting image is named `wsm/rest`.
necessary to map the source code folder and port 80 (where nginx listens):
`docker run -dit -v /var/www/wsm.rest:/var/www/wsm.rest --name wsm.rest -p 8081:80 ldesousa/wsm.rest` To run the image it is necessary to map the right environment variables to connect to the database as well as mount the source code folder and bind port 8000 (where gunicorn listens):
Gunicorn can only be started after the source code folder has been mapped. To Docker run command with envfile:
do so attach to the running container (named `wsm.rest`):
`docker attach wsm.rest` ```bash
docker run -dit \
And then run the startup script: --env-file /path/to/envfile \
-v /data/monoliths:/data/monoliths \
`sh /var/www/wsm.rest/configs/startup.sh` --name wsm.rest -p 8000:8000 wsm.rest
```
Finally, detach orderly from the running container with the escape sequence:
`Ctrl+P Ctrl+Q`
OC deployment Docker run command with variables inline:
=============
```bash
docker run -dit \
-e DB_HOST=scomp1270.wurnet.nl \
-e DB_NAME=isric \
-e DB_USER=haas005 \
-e DB_PASSWORD=<REMOVED>
-e MONOLITH_PATH=/data/monolith
-e DB_PORT=5479 \
-v /data/monoliths:/data/monoliths
--name wsm.rest -p 8000:8000 wsm.rest
```
OC uses the gitlab webhook to deploy the application on project `isric-files`. Pod application is named `rest-wsm`, and external route is `rest-wsm.isric.org`. Your container will now be available on [localhost:8000](http://localhost:8000)
Image build is based on the `python3.6` OC image `openshift/python:3.6` and WSGI system is configure with envs: ## K8s deployment
``` - `ENV` variables are set as k8s secrets on the [ISRIC deployment repository](https://git.wur.nl/isric/ict/k8s-deployments)
APP_MODULE=appWSM:app
```
Envs are stored on configuration file [here](.s2i/environment) - Application follows the standard k8s deployment.
Pod mounts the `isric-files/monoliths` on `/opt/app-root/src/monoliths (ro)` - TODO: Write about semanatic tagging
Image processing and metadata ## Image processing and metadata
=============================
Please consult the specific folder **[README](imageProcessing/README.md)** Please consult the specific folder **[README](imageProcessing/README.md)**
import sys,os
#sys.path.append('../')
sys.path.insert(0,os.path.abspath(os.path.dirname(__file__)))
from appWSM import app as application
#https://github.com/humiaozuzu/awesome-flask #https://github.com/humiaozuzu/awesome-flask
from flask import Flask,make_response from flask import Flask,make_response
from flask_restful import Api from flask_restful import Api
from flask_cors import CORS
from resources.meta import Monoliths,Monolith,MonolithRoot from resources.meta import Monoliths,Monolith,MonolithRoot
import logging,os.path import logging,os.path
app = Flask(__name__) app = Flask(__name__)
CORS(app, resources={r"/*": {'origins': '*'}})
api = Api(app) api = Api(app)
basePath=os.path.dirname(__file__) basePath=os.path.dirname(__file__)
file_handler = logging.FileHandler(os.path.join(basePath,'app.log')) #file_handler = logging.FileHandler(os.path.join(basePath,'app.log'))
app.logger.addHandler(file_handler) #app.logger.addHandler(file_handler)
app.logger.setLevel(logging.INFO) #app.logger.setLevel(logging.INFO)
app.config["LOG_TYPE"] = os.environ.get("LOG_TYPE", "stream")
app.config["LOG_LEVEL"] = os.environ.get("LOG_LEVEL", "INFO")
#'/monoliths/<path:path>', #'/monoliths/<path:path>',
api.add_resource(Monolith,"/monoliths","/") api.add_resource(Monolith,"/monoliths","/")
......
# This file must be used with "source bin/activate" *from bash*
# you cannot run it directly
deactivate () {
unset -f pydoc >/dev/null 2>&1
# reset old environment variables
# ! [ -z ${VAR+_} ] returns true if VAR is declared at all
if ! [ -z "${_OLD_VIRTUAL_PATH+_}" ] ; then
PATH="$_OLD_VIRTUAL_PATH"
export PATH
unset _OLD_VIRTUAL_PATH
fi
if ! [ -z "${_OLD_VIRTUAL_PYTHONHOME+_}" ] ; then
PYTHONHOME="$_OLD_VIRTUAL_PYTHONHOME"
export PYTHONHOME
unset _OLD_VIRTUAL_PYTHONHOME
fi
# This should detect bash and zsh, which have a hash command that must
# be called to get it to forget past commands. Without forgetting
# past commands the $PATH changes we made may not be respected
if [ -n "${BASH-}" ] || [ -n "${ZSH_VERSION-}" ] ; then
hash -r 2>/dev/null
fi
if ! [ -z "${_OLD_VIRTUAL_PS1+_}" ] ; then
PS1="$_OLD_VIRTUAL_PS1"
export PS1
unset _OLD_VIRTUAL_PS1
fi
unset VIRTUAL_ENV
if [ ! "${1-}" = "nondestructive" ] ; then
# Self destruct!
unset -f deactivate
fi
}
# unset irrelevant variables
deactivate nondestructive
VIRTUAL_ENV="/home/duque004/git/WSM/rest/appWSM"
export VIRTUAL_ENV
_OLD_VIRTUAL_PATH="$PATH"
PATH="$VIRTUAL_ENV/bin:$PATH"
export PATH
# unset PYTHONHOME if set
if ! [ -z "${PYTHONHOME+_}" ] ; then
_OLD_VIRTUAL_PYTHONHOME="$PYTHONHOME"
unset PYTHONHOME
fi
if [ -z "${VIRTUAL_ENV_DISABLE_PROMPT-}" ] ; then
_OLD_VIRTUAL_PS1="$PS1"
if [ "x" != x ] ; then
PS1="$PS1"
else
PS1="(`basename \"$VIRTUAL_ENV\"`) $PS1"
fi
export PS1
fi
# Make sure to unalias pydoc if it's already there
alias pydoc 2>/dev/null >/dev/null && unalias pydoc
pydoc () {
python -m pydoc "$@"
}
# This should detect bash and zsh, which have a hash command that must
# be called to get it to forget past commands. Without forgetting
# past commands the $PATH changes we made may not be respected
if [ -n "${BASH-}" ] || [ -n "${ZSH_VERSION-}" ] ; then
hash -r 2>/dev/null
fi
# This file must be used with "source bin/activate.csh" *from csh*.
# You cannot run it directly.
# Created by Davide Di Blasi <davidedb@gmail.com>.
alias deactivate 'test $?_OLD_VIRTUAL_PATH != 0 && setenv PATH "$_OLD_VIRTUAL_PATH" && unset _OLD_VIRTUAL_PATH; rehash; test $?_OLD_VIRTUAL_PROMPT != 0 && set prompt="$_OLD_VIRTUAL_PROMPT" && unset _OLD_VIRTUAL_PROMPT; unsetenv VIRTUAL_ENV; test "\!:*" != "nondestructive" && unalias deactivate && unalias pydoc'
# Unset irrelevant variables.
deactivate nondestructive
setenv VIRTUAL_ENV "/home/duque004/git/WSM/rest/appWSM"
set _OLD_VIRTUAL_PATH="$PATH"
setenv PATH "$VIRTUAL_ENV/bin:$PATH"
if ("" != "") then
set env_name = ""
else
set env_name = `basename "$VIRTUAL_ENV"`
endif
# Could be in a non-interactive environment,
# in which case, $prompt is undefined and we wouldn't
# care about the prompt anyway.
if ( $?prompt ) then
set _OLD_VIRTUAL_PROMPT="$prompt"
set prompt = "[$env_name] $prompt"
endif
unset env_name
alias pydoc python -m pydoc
rehash
# This file must be used using `. bin/activate.fish` *within a running fish ( http://fishshell.com ) session*.
# Do not run it directly.
function deactivate -d 'Exit virtualenv mode and return to the normal environment.'
# reset old environment variables
if test -n "$_OLD_VIRTUAL_PATH"
set -gx PATH $_OLD_VIRTUAL_PATH
set -e _OLD_VIRTUAL_PATH
end
if test -n "$_OLD_VIRTUAL_PYTHONHOME"
set -gx PYTHONHOME $_OLD_VIRTUAL_PYTHONHOME
set -e _OLD_VIRTUAL_PYTHONHOME
end
if test -n "$_OLD_FISH_PROMPT_OVERRIDE"
# Set an empty local `$fish_function_path` to allow the removal of `fish_prompt` using `functions -e`.
set -l fish_function_path
# Erase virtualenv's `fish_prompt` and restore the original.
functions -e fish_prompt
functions -c _old_fish_prompt fish_prompt
functions -e _old_fish_prompt
set -e _OLD_FISH_PROMPT_OVERRIDE
end
set -e VIRTUAL_ENV
if test "$argv[1]" != 'nondestructive'
# Self-destruct!
functions -e pydoc
functions -e deactivate
end
end
# Unset irrelevant variables.
deactivate nondestructive
set -gx VIRTUAL_ENV "/home/duque004/git/WSM/rest/appWSM"
set -gx _OLD_VIRTUAL_PATH $PATH
set -gx PATH "$VIRTUAL_ENV/bin" $PATH
# Unset `$PYTHONHOME` if set.
if set -q PYTHONHOME
set -gx _OLD_VIRTUAL_PYTHONHOME $PYTHONHOME
set -e PYTHONHOME
end
function pydoc
python -m pydoc $argv
end
if test -z "$VIRTUAL_ENV_DISABLE_PROMPT"
# Copy the current `fish_prompt` function as `_old_fish_prompt`.
functions -c fish_prompt _old_fish_prompt