Commit 37796499 authored by Wit, Allard de's avatar Wit, Allard de
Browse files

- added select on lat/lon or RD bounds

- added examples in jupyter notebook
- added version option in grompy cmd
- added --parcel_info_only
- updated the doc string
parent 3aec6d69
......@@ -126,6 +126,8 @@ run `grompy check` first.
### loading
*See information below for converting grompy 1.1 to 1.2 databases*
The final step is to load the parcel information and satellite observations into the database tables. This can be
done with the `grompy load <grompy_yaml>` command. Grompy will now show the following output:
```commandline
......@@ -147,6 +149,11 @@ server with sufficient capacity to handle multiple streams of data. Note that gr
into one SQLite database, but write locks on the database will cause delays in processing so this is not
recommended.
*Important*: grompy 1.2 offers additional parcel selection options compared to grompy 1.2. To convert a grompy 1.1 database
to a grompy 1.2 database it is only necessary to reload the parcel information. The structure of the Sentinel1/2
parcel observations did not change from 1.1 to 1.2. A special option has been added to only reload the parcel
info in a grompy database: `grompy load --parcel_info_only <grompy_yaml>`
## Accessing data processed by grompy
See the jupyter notebooks in the `notebooks/` directory in the grompy git repository.
\ No newline at end of file
# -*- coding: utf-8 -*-
# Copyright (c) 2021 Wageningen Environmental Research
# Allard de Wit (allard.dewit@wur.nl), April 2021
# Allard de Wit (allard.dewit@wur.nl), October 2021
"""Grompy is a tool to process and access parcel-based satellite observations from GroenMonitor.nl.
"""
__version__ = "1.2.0"
from .dap import DataAccessProvider
from .cmd import cli
__version__ = "1.1.1"
\ No newline at end of file
......@@ -8,6 +8,7 @@ import click
from .load_data import load_data
from .check_files import check_grompy_inputs
from .init import init
from . import __version__
@click.group()
def cli():
......@@ -35,9 +36,17 @@ def check_grompy(grompy_yaml):
@click.command("load")
@click.argument("grompy_yaml", type=click.Path(exists=True))
def load_grompy(grompy_yaml):
@click.option("--parcel_info_only", is_flag=True)
def load_grompy(grompy_yaml, parcel_info_only):
grompy_yaml = Path(grompy_yaml)
load_data(grompy_yaml)
load_data(grompy_yaml, parcel_info_only)
@click.group()
@click.version_option(__version__)
@click.pass_context
def cli(ctx):
pass
cli.add_command(init_grompy)
......
......@@ -15,9 +15,27 @@ class Container:
class DataAccessProvider:
"""grompy.DataAccessProvider allow to query grompy databases and iterate through the selected parcels.
:param grompy_yaml: path to the grompy YAML file
:param fieldID: A list of fieldIDs that should be selected. Note that the number of fieldIDs that can be
provided is limited but unknown (depending on the SQL parser). Above 1000 FieldIDs warning
messages will be displayed. When this keyword is used, all other selection criteria will
be ignored.
:param area_gt: Only selected parcels greater than X ha
:param pixcount_gt: Only select parcels with a pixel count greater than this number
:param provincie: Only select parcels within this province
:param limit: Limit the number of parcels selected.
:param gws_gewasc: Select only parcels with given gws_gewasc
:param cat_gewasc: Select only parcels with given cat_gewasc
:param rdx_bounds: Select only parcels within the [min, max] RD X coordinates
:param rdy_bounds: Select only parcels within the [min, max] RD Y coordinates
:param lon_bounds: Select only parcels within the [min, max] longitude
:param lat_bounds: Select only parcels within the [min, max] latitude
"""
def __init__(self, grompy_yaml, fieldID=None, area_gt=None, pixcount_gt=None, provincie=None, limit=None,
gws_gewasc=None, cat_gewasc=None):
gws_gewasc=None, cat_gewasc=None, rdx_bounds=None, rdy_bounds=None, lon_bounds=None, lat_bounds=None):
grompy_yaml = Path(grompy_yaml)
if not grompy_yaml.exists():
......@@ -64,6 +82,14 @@ class DataAccessProvider:
if cat_gewasc is not None:
cat_gewasc = str(cat_gewasc).strip()
self.perc_stmt.append_whereclause(self.tbl_perc_info.c.cat_gewasc == cat_gewasc)
if rdx_bounds is not None:
self.perc_stmt.append_whereclause(sa.between(self.tbl_perc_info.c.rdx, rdx_bounds[0], rdx_bounds[1]))
if rdy_bounds is not None:
self.perc_stmt.append_whereclause(sa.between(self.tbl_perc_info.c.rdy, rdy_bounds[0], rdy_bounds[1]))
if lon_bounds is not None:
self.perc_stmt.append_whereclause(sa.between(self.tbl_perc_info.c.longitude, lon_bounds[0], lon_bounds[1]))
if lat_bounds is not None:
self.perc_stmt.append_whereclause(sa.between(self.tbl_perc_info.c.latitude, lat_bounds[0], lat_bounds[1]))
s = sa.select([sa.func.count()]).select_from(self.perc_stmt)
self.parcel_count = s.execute().fetchone()[0]
......@@ -112,4 +138,3 @@ class DataAccessProvider:
def __len__(self):
return self.parcel_count
......@@ -241,7 +241,7 @@ def monitor_parallel_loading(process_list, parent_conn, lines_per_dataset):
print(e.traceback)
def load_data(grompy_yaml):
def load_data(grompy_yaml, parcel_info_only):
"""Loads data point to by the YAML config file.
:param grompy_yaml:
......@@ -255,5 +255,6 @@ def load_data(grompy_yaml):
parcel_info = grompy_conf.pop("parcel_info")
load_parcel_info(grompy_yaml, **parcel_info)
process_list, parent_conn, lines_per_dataset = start_parallel_loading(grompy_yaml, grompy_conf["datasets"])
monitor_parallel_loading(process_list, parent_conn, lines_per_dataset)
if not parcel_info_only:
process_list, parent_conn, lines_per_dataset = start_parallel_loading(grompy_yaml, grompy_conf["datasets"])
monitor_parallel_loading(process_list, parent_conn, lines_per_dataset)
......@@ -130,8 +130,8 @@ def open_DB_connection(grompy_yaml, dsn):
def get_latlon_from_RD(x, y, _cache={}):
rd_stelsel = "epsg:7415"
try:
p1 = _cache["rd2wgs84"]
return p1(x, y, inverse=True)
rd2wgs84 = _cache["rd2wgs84"]
except KeyError:
p1 = _cache["rd2wgs84"] = pyproj.Proj(init=rd_stelsel)
return p1(x, y, inverse=True)
rd2wgs84 = _cache["rd2wgs84"] = pyproj.Proj(init=rd_stelsel)
return rd2wgs84(x, y, inverse=True)
This diff is collapsed.
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment