Initial commit

.gitlab-ci.yml

+image:
+  name: munlock/build
+  entrypoint: [""] # force an empty entrypoint
+
+#stages to execute
+stages:
+- test_build
+
+before_script:
+  - apt-get update
+  - apt-get install -y lftp
+  - export PATH="/root/.sdkman/candidates/maven/current/bin:/root/.sdkman/candidates/java/current/bin:/root/.sdkman/candidates/gradle/current/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"
+
+#test the gradle build
+test_build_dev:
+  stage: test_build
+  script:
+    - echo $0
+    - echo $PATH
+    - ./build.sh
+    - lftp sftp://$USERNAME:$PASSWORD@$HOST -e "set sftp:auto-confirm yes; cd upload/unlock/;put ./iRODSKubernetes.jar; bye"
+  only:
+    - dev
+
+#test the gradle build
+test_build_master:
+  stage: test_build
+  script:
+    - echo $0
+    - echo $PATH
+    - ./build.sh
+    - lftp sftp://$USERNAME:$PASSWORD@$HOST -e "set sftp:auto-confirm yes; cd upload/unlock/;put ./iRODSKubernetes.jar; bye"
+  only:
+    - master
\ No newline at end of file

build.gradle

+/*
+ * This file was generated by the Gradle 'init' task.
+ *
+ * This generated file contains a sample Java project to get you started.
+ * For more details take a look at the Java Quickstart chapter in the Gradle
+ * user guide available at https://docs.gradle.org/5.0/userguide/tutorial_java_projects.html
+ */
+
+buildscript {
+    repositories {
+        mavenCentral()
+        mavenLocal()
+        maven { url 'https://repo.gradle.org/gradle/libs-releases' }
+        jcenter()
+        maven { url "https://plugins.gradle.org/m2/" }
+    }
+    dependencies {
+        // Jar
+        classpath "com.github.jengelman.gradle.plugins:shadow:6.0.0"
+    }
+}
+
+plugins {
+    // Apply the java plugin to add support for Java
+    id 'java'
+
+    // Apply the application plugin to add support for building an application
+    id 'application'
+}
+
+apply plugin: "com.github.johnrengelman.shadow"
+
+shadowJar {
+    baseName = 'iRODSKubernetes'
+    classifier = null
+    version = null
+    configurations = [project.configurations.runtimeClasspath]
+
+}
+
+jar {
+    manifest {
+        attributes 'Main-Class': 'nl.wur.ssb.App'
+    }
+}
+
+artifacts {
+    archives shadowJar
+}
+
+group = 'nl.munlock.kubernetes'
+version = '0.1.0'
+description = """Kubernetes manager for unlock"""
+
+repositories {
+    // Use jcenter for resolving your dependencies.
+    // You can declare any Maven/Ivy/file repository here.
+    jcenter()
+    mavenLocal()
+}
+
+dependencies {
+    // https://mvnrepository.com/artifact/com.google.guava/guava
+    compile group: 'com.google.guava', name: 'guava', version: '27.1-jre'
+
+    // This dependency is found on compile classpath of this component and consumers.
+    implementation 'com.google.guava:guava:26.0-jre'
+
+    // Use JUnit test framework
+    testImplementation 'junit:junit:4.12'
+
+    // https://mvnrepository.com/artifact/org.apache.poi/poi
+    compile group: 'org.apache.poi', name: 'poi', version: '4.1.0'
+
+    // https://mvnrepository.com/artifact/org.apache.poi/poi-ooxml
+    compile group: 'org.apache.poi', name: 'poi-ooxml', version: '4.1.0'
+
+    // https://mvnrepository.com/artifact/com.github.samtools/htsjdk
+    compile group: 'com.github.samtools', name: 'htsjdk', version: '2.19.0'
+
+    // https://mvnrepository.com/artifact/com.jcraft/jsch
+    compile group: 'com.jcraft', name: 'jsch', version: '0.1.55'
+
+    // Kubernetes api
+    compile 'io.kubernetes:client-java:5.0.0'
+
+    // Logger
+    compile group: 'log4j', name: 'log4j', version:'1.2.17'
+
+    // https://mvnrepository.com/artifact/com.esotericsoftware.yamlbeans/yamlbeans
+    compile group: 'com.esotericsoftware.yamlbeans', name: 'yamlbeans', version: '1.14'
+
+    // locally installed jargon and unlock api
+    compile group: 'jargon', name: 'core', version: '4.3.0.2'
+    compile group: 'nl.munlock', name: 'unlockapi', version: '1.0.1'
+}
+
+// Define the main class for the application
+mainClassName = 'nl.wur.ssb.App'
\ No newline at end of file

build.sh

+#!/bin/bash
+#============================================================================
+#title          :IRODS CWL
+#description    :IRODS manager installation script
+#author         :Jasper Koehorst
+#date           :2019
+#version        :0.0.1
+#============================================================================
+
+DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
+
+git -C $DIR pull
+
+# ////////////////////////////////////////////////////////////////////////////////////
+# Jargon dependency
+# ////////////////////////////////////////////////////////////////////////////////////
+
+wget -nc https://github.com/DICE-UNC/jargon/releases/download/4.3.0.2-RELEASE/jargon-core-4.3.0.2-RELEASE-jar-with-dependencies.jar -O $DIR/jargon-core-4.3.0.2-RELEASE-jar-with-dependencies.jar
+mvn install:install-file -Dfile=$DIR/jargon-core-4.3.0.2-RELEASE-jar-with-dependencies.jar -DgroupId=jargon -DartifactId=core -Dversion=4.3.0.2 -Dpackaging=jar
+
+# ////////////////////////////////////////////////////////////////////////////////////
+# // UNLOCK API
+# ////////////////////////////////////////////////////////////////////////////////////
+wget -nc http://download.systemsbiology.nl/unlock/UnlockOntology.jar -O $DIR/UnlockOntology.jar
+mvn install:install-file -Dfile=$DIR/UnlockOntology.jar -DgroupId=nl.munlock -DartifactId=unlockapi -Dversion=1.0.1 -Dpackaging=jar
+
+# Building mode
+if [ "$1" == "test" ]; then
+	gradle build -b "$DIR/build.gradle" --info
+else
+	echo "Skipping tests, run './install.sh test' to perform tests"
+	gradle build -b "$DIR/build.gradle" -x test
+fi
+
+cp $DIR/build/libs/*jar $DIR/
+
+# java -jar $DIR/IRODSTransfer.jar --help

settings.gradle

+/*
+ * This file was generated by the Gradle 'init' task.
+ *
+ * The settings file is used to specify which projects to include in your build.
+ * 
+ * Detailed information about configuring a multi-project build in Gradle can be found
+ * in the user guide at https://docs.gradle.org/4.5.1/userguide/multi_project_builds.html
+ */
+
+rootProject.name = 'iRODSKubernetes'

src/main/java/nl/munlock/App.java

+package nl.munlock;
+
+import nl.munlock.yaml.Yaml;
+import org.apache.log4j.Logger;
+
+import java.util.Arrays;
+
+public class App {
+    /**
+     * Main landing point for yaml or kubernetes runs...
+     * @param args
+     * @throws Exception
+     */
+    private static final Logger log = Generic.getLogger(Yaml.class, false);
+
+    public static void main(String[] args) throws Exception {
+        if (Arrays.asList(args).contains("-cwl")) {
+            log.info("Generating YAML files");
+            Yaml.main(args);
+        } else if (Arrays.asList(args).contains("-kubernetes")) {
+            log.info("Executing kubernetes workflows");
+            nl.munlock.kubernetes.App.main(args);
+        }
+    }
+}

src/main/java/nl/munlock/Generic.java

+package nl.munlock;
+
+import nl.munlock.irods.Connection;
+import org.apache.commons.io.FilenameUtils;
+import org.apache.log4j.*;
+import org.irods.jargon.core.exception.JargonException;
+import org.irods.jargon.core.pub.DataTransferOperations;
+import org.irods.jargon.core.pub.io.IRODSFile;
+import org.slf4j.LoggerFactory;
+
+import java.io.File;
+import java.io.FileNotFoundException;
+import java.util.Scanner;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+public class Generic {
+    private static final org.slf4j.Logger log = LoggerFactory.getLogger(Generic.class);
+
+    /**
+     * Logger initialization with debug option
+     * @param debug boolean if debug mode should be enabled
+     *
+     * @return
+     */
+    public static Logger getLogger(Class clazz, boolean debug) {
+        ConsoleAppender console = new ConsoleAppender();
+        String PATTERN = "%d %-5p [%c{1}] %m%n";
+        console.setLayout(new PatternLayout(PATTERN));
+        console.setThreshold(Level.DEBUG);
+        console.activateOptions();
+
+        Logger.getRootLogger().removeAllAppenders();
+        Logger.getRootLogger().addAppender(console);
+
+        Logger logger = Logger.getLogger(clazz);
+
+        logger.setLevel(Level.INFO);
+        if (debug)
+            logger.setLevel(Level.DEBUG);
+
+
+        FileAppender fa = new FileAppender();
+        fa.setName("iRODS Logger");
+        fa.setFile("runner.log");
+        fa.setLayout(new PatternLayout("%d %-5p [%c{1}] %m%n"));
+        fa.setThreshold(Level.INFO);
+        if (debug) fa.setThreshold(Level.DEBUG);
+
+        fa.setAppend(true);
+        fa.activateOptions();
+        org.apache.log4j.Logger.getRootLogger().addAppender(fa);
+        return logger;
+    }
+
+    public static String getAssayPath(File collection) throws FileNotFoundException {
+        String regex = "(.*/S_.*?/(Amplicon|RNA|DNA)/A_.*?)/";
+        Pattern pattern = Pattern.compile(regex);
+        Matcher matcher = pattern.matcher(collection.getAbsolutePath() + "/");
+        while (matcher.find()) {
+            return matcher.group(1);
+        }
+        throw new FileNotFoundException("Assay path not found in " + collection);
+    }
+
+    public static void downloadFile(Connection connection, File download) throws JargonException {
+        IRODSFile irodsFile = connection.fileFactory.instanceIRODSFile(download.getAbsolutePath());
+        if (!irodsFile.exists()) {
+            throw new JargonException("File " + irodsFile + " does not exist");
+        }
+        org.apache.log4j.Logger.getLogger("org.nl.wur.ssb.irods.jargon.core.transfer").setLevel(Level.OFF);
+        DataTransferOperations dataTransferOperationsAO = connection.irodsFileSystem.getIRODSAccessObjectFactory().getDataTransferOperations(connection.irodsAccount);
+
+        // Create folder directory
+        File directory = new File("." + download.getAbsolutePath().replaceAll(download.getName() + "$", ""));
+        directory.mkdirs();
+
+        File localFile = new File("." + download);
+
+        while (localFile.exists()) {
+            localFile.delete();
+        }
+
+        // Disables the logger for the transfer as it easily gives thousands of lines...
+        org.apache.log4j.Logger.getLogger("org.nl.wur.ssb.irods.jargon.core.transfer").setLevel(Level.OFF);
+        log.info("Downloading " + localFile.getName());
+
+        dataTransferOperationsAO.getOperation(irodsFile, localFile, null, null);
+    }
+
+    public static void destinationChecker(Connection connection, File file) {
+        Scanner scanner = new Scanner(file.getAbsolutePath());
+        while (scanner.hasNextLine()) {
+            String line = scanner.nextLine();
+            System.err.println(line);
+            if (line.startsWith("destination")) {
+                System.err.println(line);
+            }
+        }
+    }
+
+    public static String getEdamFormat(String fileName) throws Exception {
+        // Always remove compression end when gz
+        fileName = fileName.replaceAll(".gz$","");
+
+        String extension = FilenameUtils.getExtension(fileName);
+
+        log.info("File extension " + extension + " detected");
+        if (fileName.startsWith("http")) {
+            return fileName;
+        } else if (extension.matches("bam")) {
+            return "http://edamontology.org/format_2572";
+        } else if (extension.matches("(fq|fastq)")) {
+            return "http://edamontology.org/format_1930";
+        } else if (extension.matches("(fa|fasta)")) {
+            return "http://edamontology.org/format_1929";
+        } else {
+            log.error("Unknown file format " + extension);
+            return null;
+        }
+    }
+}

src/main/java/nl/munlock/irods/Connection.java

+package nl.munlock.irods;
+
+import nl.munlock.options.irods.CommandOptionsIRODS;
+import nl.munlock.options.kubernetes.CommandOptionsKubernetes;
+import org.irods.jargon.core.connection.ClientServerNegotiationPolicy;
+import org.irods.jargon.core.connection.IRODSAccount;
+import org.irods.jargon.core.exception.JargonException;
+import org.irods.jargon.core.pub.IRODSAccessObjectFactory;
+import org.irods.jargon.core.pub.IRODSFileSystem;
+import org.irods.jargon.core.pub.io.IRODSFileFactory;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import static org.irods.jargon.core.connection.ClientServerNegotiationPolicy.SslNegotiationPolicy.CS_NEG_REFUSE;
+import static org.irods.jargon.core.connection.ClientServerNegotiationPolicy.SslNegotiationPolicy.CS_NEG_REQUIRE;
+
+public class Connection {
+    public final IRODSFileSystem irodsFileSystem;
+    public final IRODSAccount irodsAccount;
+    public final IRODSAccessObjectFactory accessObjectFactory;
+    public final IRODSFileFactory fileFactory;
+    private static final Logger log = LoggerFactory.getLogger(Connection.class);
+
+    public Connection(CommandOptionsIRODS commandOptions) throws JargonException {
+
+        // Initialize account object
+        irodsAccount = IRODSAccount.instance(commandOptions.host,Integer.parseInt(commandOptions.port),commandOptions.username, commandOptions.password,"",commandOptions.zone,"");
+
+        // set SSL settings
+        ClientServerNegotiationPolicy clientServerNegotiationPolicy = new ClientServerNegotiationPolicy();
+        ClientServerNegotiationPolicy.SslNegotiationPolicy sslPolicy;
+        if (commandOptions.sslPolicyString.matches("CS_NEG_REQUIRE")) {
+            sslPolicy = CS_NEG_REQUIRE;
+        } else if (commandOptions.sslPolicyString.matches("CS_NEG_REFUSE")) {
+            sslPolicy = CS_NEG_REFUSE;
+        } else {
+            throw new JargonException("SSL policy not recognised: " + commandOptions.sslPolicyString);
+        }
+
+        clientServerNegotiationPolicy.setSslNegotiationPolicy(sslPolicy);
+        irodsAccount.setClientServerNegotiationPolicy(clientServerNegotiationPolicy);
+
+        irodsFileSystem = IRODSFileSystem.instance();
+
+        accessObjectFactory = irodsFileSystem.getIRODSAccessObjectFactory();
+
+//        AuthResponse x = accessObjectFactory.authenticateIRODSAccount(irodsAccount);
+//        log.info(x.getAuthMessage());
+
+        fileFactory = accessObjectFactory.getIRODSFileFactory(irodsAccount);
+    }
+}

src/main/java/nl/munlock/kubernetes/App.java

+package nl.munlock.kubernetes;
+
+import nl.munlock.options.kubernetes.CommandOptionsKubernetes;
+import nl.munlock.irods.Connection;
+import org.apache.log4j.Level;
+
+public class App {
+    public static void main(String[] args) throws Exception {
+        org.apache.log4j.Logger.getLogger("org").setLevel(Level.OFF);
+
+        CommandOptionsKubernetes commandOptions = new CommandOptionsKubernetes(args);
+        Connection connection = new Connection(commandOptions);
+        Kubernetes.main(commandOptions, connection);
+    }
+}

src/main/java/nl/munlock/kubernetes/Kubernetes.java

+package nl.munlock.kubernetes;
+
+import com.esotericsoftware.yamlbeans.YamlException;
+import com.esotericsoftware.yamlbeans.YamlReader;
+import com.google.gson.JsonSyntaxException;
+import io.kubernetes.client.ApiClient;
+import io.kubernetes.client.ApiException;
+import io.kubernetes.client.Configuration;
+import io.kubernetes.client.apis.BatchV1Api;
+import io.kubernetes.client.apis.CoreV1Api;
+import io.kubernetes.client.custom.Quantity;
+import io.kubernetes.client.models.*;
+import io.kubernetes.client.util.Config;
+import nl.munlock.Generic;
+import nl.munlock.irods.Connection;
+import nl.munlock.irods.Search;
+import nl.munlock.options.kubernetes.CommandOptionsKubernetes;
+import nl.munlock.objects.Workflow;
+import org.apache.log4j.Logger;
+import org.irods.jargon.core.exception.JargonException;
+import org.irods.jargon.core.query.*;
+
+import java.io.File;
+import java.io.FileNotFoundException;
+import java.io.FileReader;
+import java.io.IOException;
+import java.util.*;
+import java.util.concurrent.TimeUnit;
+
+import static nl.munlock.yaml.Workflow.fixClass;
+
+public class Kubernetes {
+    private static final Logger log = Generic.getLogger(Kubernetes.class, false);
+    public static HashSet<String> yamls = new HashSet<>();
+
+    public static void main(CommandOptionsKubernetes commandOptionsKubernetes, Connection connection) throws Exception {
+        getYamls(commandOptionsKubernetes, connection);
+        createJobs(commandOptionsKubernetes, connection);
+    }
+
+    public static void createJobs(CommandOptionsKubernetes commandOptionsKubernetes, Connection connection) throws IOException, ApiException, JargonQueryException, JargonException, InterruptedException {
+        // Requires the kube config file
+        ApiClient client = Config.defaultClient();
+        client.setDebugging(false);
+        Configuration.setDefaultApiClient(client);
+
+        int assayCount = 0;
+        // Check nl.wur.ssb.kubernetes if it has room for more jobs...
+        int totalItems = checkKubernetesJobs(client);
+
+        for (String yaml : yamls) {
+
+            assayCount = assayCount + 1;
+
+            V1Job v1Job = generateKubernetesJobObject(commandOptionsKubernetes, yaml, connection);
+
+            // Sleeping for 1 minute...
+            while (totalItems >= commandOptionsKubernetes.limit) {
+                log.info("Sleeping for 1 minute as there are " + totalItems + " jobs running");
+                // Count down?
+                for (int i = 60; i >= 0; i--) {
+                    System.out.print("Sleeping for " + i + " seconds\r");
+                    TimeUnit.SECONDS.sleep(1);
+                }
+                totalItems = checkKubernetesJobs(client);
+            }
+
+            // v1Job.setKind("my-kind");
+            BatchV1Api batchV1Api = new BatchV1Api();
+
+            // Submit the job to nl.wur.ssb.kubernetes
+            batchV1Api.createNamespacedJob("unlock", v1Job, false, null, null);
+            batchV1Api.createNamespacedJobCall("unlock", v1Job, null, null, null, null, null);
+
+            TimeUnit.MILLISECONDS.sleep(500);
+
+            totalItems = totalItems + 1;
+
+            int remain = yamls.size() - assayCount;
+            log.info("Jobs currently running " + totalItems + " of " + remain);
+        }
+        yamls = new HashSet<>();
+    }
+
+    private static void getYamls(CommandOptionsKubernetes commandOptionsKubernetes, Connection connection) throws JargonException, GenQueryBuilderException, JargonQueryException, IOException, ApiException, InterruptedException {
+        // List all project turtle files...
+        Search.getAllUnprocessed(commandOptionsKubernetes, connection);
+        Kubernetes.createJobs(commandOptionsKubernetes, connection);
+        Search.getAllUnprocessedReferences(commandOptionsKubernetes, connection);
+        Kubernetes.createJobs(commandOptionsKubernetes, connection);
+        Search.getAllYamls(commandOptionsKubernetes, connection);
+        Kubernetes.createJobs(commandOptionsKubernetes, connection);
+    }
+
+
+    /**
+     * Only manages the irodsrunner- jobs... and cleans them up when needed...
+     *
+     * @return
+     * @throws ApiException
+     * @param client
+     */
+    public static int checkKubernetesJobs(ApiClient client) throws ApiException {
+
+        BatchV1Api batchV1Api = new BatchV1Api(client);
+
+        List<V1Job> v1Jobs = batchV1Api.listNamespacedJob("unlock", null, null, null, null, null, null, null, null, null).getItems();
+
+        int totalJobs = v1Jobs.size();
+        log.info("Total jobs before removal " + totalJobs);
+        Iterator<V1Job> v1JobIterator = v1Jobs.iterator();
+        while (v1JobIterator.hasNext()) {
+            V1Job v1Job = v1JobIterator.next();
+            if (v1Job.getMetadata().getName().startsWith("cwl-")) {
+                if (v1Job.getStatus().getActive() == null) {
+                    log.info("Deleting " + v1Job.getMetadata().getName() + " " + totalJobs);
+                    try {
+                        batchV1Api.deleteNamespacedJob(v1Job.getMetadata().getName(), "unlock", null, null, null, null, null, null);
+                    } catch (com.google.gson.JsonSyntaxException | io.kubernetes.client.ApiException e) {
+                        if (e.getMessage().contains("Expected a string but was BEGIN_OBJECT")) {
+                            // Exception thrown hen item is being deleted, api is waiting for new implementation...
+                        } else if (e.getMessage().contains("Not Found")) {
+                            // Job was already on the way to deletion and we went to fast?
+                        } else {
+                            throw new JsonSyntaxException(e.getMessage());
+                        }
+                        totalJobs = totalJobs - 1;
+                    }
+                }
+            }
+        }
+
+        // Remove pods as TTL is not always working when finished
+        CoreV1Api api = new CoreV1Api();
+        V1PodList v1PodList = api.listPodForAllNamespaces(null, null, null, null, null, null, null, null, null);
+        Iterator<V1Pod> v1PodIterator = v1PodList.getItems().iterator();
+        while (v1PodIterator.hasNext()) {
+            V1Pod v1Pod = v1PodIterator.next();
+            if (v1Pod.getMetadata().getName().startsWith("cwl-")) {
+                if (v1Pod.getStatus().getPhase().matches("Succeeded")) {
+                    log.info("Deleting " + v1Pod.getMetadata().getName() + " " + totalJobs);
+                    try {
+                        api.deleteNamespacedPod(v1Pod.getMetadata().getName(), v1Pod.getMetadata().getNamespace(), null, null, null, null, null, null);
+                    } catch (com.google.gson.JsonSyntaxException | io.kubernetes.client.ApiException e) {
+                        if (e.getMessage().contains("Expected a string but was BEGIN_OBJECT")) {
+                            // Exception thrown hen item is being deleted, api is waiting for new implementation...
+                        } else if (e.getMessage().contains("Not Found")) {
+                            // Job was already on the way to deletion and we went to fast?
+                        } else {
+                            throw new JsonSyntaxException(e.getMessage());
+                        }
+                        totalJobs = totalJobs - 1;
+                    }
+                }
+            }
+        }
+
+        if (totalJobs < 0) {
+            totalJobs = 0;
+        }
+
+        log.info("Jobs currently running " + totalJobs);
+
+        return totalJobs;
+    }
+
+    private static V1Job generateKubernetesJobObject(CommandOptionsKubernetes commandOptions, String yamlFile, Connection connection) throws JargonException, JargonQueryException, YamlException, FileNotFoundException {
+
+        // Load yaml file for cores and memory restrictions?
+        log.info("Loading ." + yamlFile);
+        fixClass("." + yamlFile);
+        YamlReader reader = new YamlReader(new FileReader("." + yamlFile));
+        // Hack as ngtax currently cannot handle the -!nl.... paths in the yaml file
+        Workflow workflow;
+        try {
+            workflow = (Workflow) reader.read();
+        } catch (YamlReader.YamlReaderException e) {
+            Scanner scanner = new Scanner(new File("." + yamlFile));
+            workflow = new Workflow();
+            while (scanner.hasNextLine()) {
+                String line = scanner.nextLine();
+                if (line.startsWith("threads")) {
+                    workflow.threads = Integer.parseInt(line.split(" ")[1]);
+                }
+                if (line.startsWith("memory")) {
+                    workflow.memory = Integer.parseInt(line.split(" ")[1]);
+                }
+            }
+        }
+
+        if (workflow.threads == 0) {