First version of csvtools include documentation and test

README.md

 # csvtools
 
-csvtools is a tool to proces CSV files
\ No newline at end of file
+csvtools is a tool to proces CSV files
+
+## Requirements
+
+For installing the csvtools package we have the following dependencies:
+- pandas >= 0.13
+- sqlalchemy >= 1.3
+- click >= 8.0
+
+## install
+
+Installing csvtools can be done with pip through:
+```commandline
+pip install csvtools
+```
+
+
+# How to use
+
+```commandline
+Usage: python -m csvtools.cmd [OPTIONS] COMMAND [ARGS]...
+
+  With csvtools you can operate on CSV files
+
+Options:
+  --help  Show this message and exit.
+
+Commands:
+  csv2db
+
+```
\ No newline at end of file

csvtools/__init__.py

+# -*- coding: utf-8 -*-
+# Copyright (c) 2022, Wageningen Environmental Research
+# Allard de Wit (allard.dewit@wur.nl), January 2022
+"""csvtools is a set of utilities to process CSV files
+"""
+__version__ = "0.9.0"
\ No newline at end of file

csvtools/cmd.py

+# -*- coding: utf-8 -*-
+# Copyright (c) 2022, Wageningen Environmental Research
+# Allard de Wit (allard.dewit@wur.nl), January 2022
 import click
 
 from .csv2db import process_csv
@@ -12,6 +15,12 @@ def cli():
 @click.argument("csv_fname", type=click.Path(exists=True))
 @click.argument("sqlite_fname", type=click.Path())
 def cmd_csv2db(csv_fname, sqlite_fname):
+    """Function for the commandline interface for csv2db
+
+    :param csv_fname: the CSV input filename
+    :param sqlite_fname: the SQLite output filename
+
+    """
     process_csv(csv_fname, sqlite_fname)
 
 

csvtools/csv2db.py

+# -*- coding: utf-8 -*-
+# Copyright (c) 2022, Wageningen Environmental Research
+# Allard de Wit (allard.dewit@wur.nl), January 2022
 from pathlib import Path
 
 import pandas as pd
@@ -5,6 +8,11 @@ import sqlalchemy as sa
 
 
 def get_database_engine(sqlite_fname):
+    """Generates the SQLalchemy connection to the database
+
+    :param sqlite_fname: the SQLite filename
+    :return: a SA database connection
+    """
 
     dsn = f"sqlite:///{sqlite_fname}"
     engine = sa.create_engine(dsn)
@@ -12,6 +20,12 @@ def get_database_engine(sqlite_fname):
 
 
 def process_csv(csv_fname, sqlite_fname):
+    """Writes a CSV file into a SQLite database file
+
+    :param csv_fname: Filename of the CSV file
+    :param sqlite_fname: Filename of the SQLite database
+    :return: a pandas dataframe
+    """
     csv_fname = Path(csv_fname).absolute()
     sqlite_fname = Path(sqlite_fname).absolute()
 

pyproject.toml

+[build-system]
+requires = ["flit_core >=3.2,<4"]
+build-backend = "flit_core.buildapi"
+
+[project]
+name = "csvtools"
+authors = [{name = "Allard de Wit", email = "allard.dewit@wur.nl"}]
+readme = "README.md"
+dynamic = ["version", "description"]
+requires-python = ">=3.8"
+dependencies = [
+    "pandas >= 1.2",
+    "sqlalchemy >= 1.3",
+    "click >= 8.0"
+]
+
+[project.urls]
+Home = "https://www.wur.nl"
+
+[project.scripts]
+csvtools = "csvtools.cmd:main"

run/run_csv2db.py

+# -*- coding: utf-8 -*-
+# Copyright (c) 2022, Wageningen Environmental Research
+# Allard de Wit (allard.dewit@wur.nl), January 2022
 from csvtools.csv2db import process_csv
 
 csv_fname = r"c:\temp\US_UFO_sightings.csv"

test/csv2db_test.py

+import unittest
+from pathlib import Path
+import tempfile
+
+import pandas as pd
+import sqlalchemy as sa
+
+from csvtools.csv2db import process_csv, get_database_engine
+
+
+class TestCSV2DB(unittest.TestCase):
+
+    def runTest(self):
+        this_dir = Path(__file__).parent
+        test_data = this_dir / "testdata.csv"
+        test_db = Path(tempfile.gettempdir()) / "testdata.db"
+
+        df1 = process_csv(test_data, test_db)
+
+        # read back CSV from database
+        engine = get_database_engine(test_db)
+        df2 = pd.read_sql_table("csv_data", engine)
+
+        self.assertEqual(len(df1), len(df2))
+        for row1, row2 in zip(df1.itertuples(), df2.itertuples()):
+            self.assertEqual(row1, row2)

test/testdata.csv

+datetime,city,state,country,shape,duration_seconds,duration_hours_mins,comments,date_posted,latitude,longitude
+10/10/1949 20:30,san marcos,tx,us,cylinder,2700,45 minutes,This event took place in early fall around 1949-50. It occurred after a Boy Scout meeting in the Baptist Church. The Baptist Church sit,4/27/2004,29.8830556,-97.9411111
+10/10/1949 21:00,lackland afb,tx,XX,light,7200,1-2 hrs,1949 Lackland AFB&#44 TX.  Lights racing across the sky & making 90 degree turns on a dime.,12/16/2005,29.38421,-98.581082
+10/10/1955 17:00,chester (uk/england),XX,gb,circle,20,20 seconds,Green/Orange circular disc over Chester&#44 England,1/21/2008,53.2,-2.916667
+10/10/1956 21:00,edna,tx,us,circle,20,1/2 hour,My older brother and twin sister were leaving the only Edna theater at about 9 PM&#44...we had our bikes and I took a different route home,1/17/2004,28.9783333,-96.6458333
+10/10/1960 20:00,kaneohe,hi,us,light,900,15 minutes,AS a Marine 1st Lt. flying an FJ4B fighter/attack aircraft on a solo night exercise&#44 I was at 50&#44000&#39 in a "clean" aircraft (no ordinan,1/22/2004,21.4180556,-157.8036111
+10/10/1961 19:00,bristol,tn,us,sphere,300,5 minutes,My father is now 89 my brother 52 the girl with us now 51 myself 49 and the other fellow which worked with my father if he&#39s still livi,4/27/2007,36.595,-82.1888889
+10/10/1965 21:00,penarth (uk/wales),XX,gb,circle,180,about 3 mins,penarth uk  circle  3mins  stayed 30ft above me for 3 mins slowly moved of and then with the blink of the eye the speed was unreal,2/14/2006,51.434722,-3.18
+10/10/1965 23:45,norwalk,ct,us,disk,1200,20 minutes,A bright orange color changing to reddish color disk/saucer was observed hovering above power transmission lines.,10/2/1999,41.1175,-73.4083333
+10/10/1966 20:00,pell city,al,us,disk,180,3  minutes,Strobe Lighted disk shape object observed close&#44 at low speeds&#44 and low altitude in Oct 1966 in Pell City Alabama,3/19/2009,33.5861111,-86.2861111