Commit ebb4159e authored by Koehorst, Jasper's avatar Koehorst, Jasper
Browse files

Initial commit

parents
Pipeline #14533 failed with stage
image:
name: munlock/build
entrypoint: [""] # force an empty entrypoint
#stages to execute
stages:
- test_build
before_script:
- apt-get update
- apt-get install -y lftp
- export PATH="/root/.sdkman/candidates/maven/current/bin:/root/.sdkman/candidates/java/current/bin:/root/.sdkman/candidates/gradle/current/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"
#test the gradle build
test_build_dev:
stage: test_build
script:
- echo $0
- echo $PATH
- ./build.sh
- lftp sftp://$USERNAME:$PASSWORD@$HOST -e "set sftp:auto-confirm yes; cd upload/unlock/;put ./iRODSKubernetes.jar; bye"
only:
- dev
#test the gradle build
test_build_master:
stage: test_build
script:
- echo $0
- echo $PATH
- ./build.sh
- lftp sftp://$USERNAME:$PASSWORD@$HOST -e "set sftp:auto-confirm yes; cd upload/unlock/;put ./iRODSKubernetes.jar; bye"
only:
- master
\ No newline at end of file
/*
* This file was generated by the Gradle 'init' task.
*
* This generated file contains a sample Java project to get you started.
* For more details take a look at the Java Quickstart chapter in the Gradle
* user guide available at https://docs.gradle.org/5.0/userguide/tutorial_java_projects.html
*/
buildscript {
repositories {
mavenCentral()
mavenLocal()
maven { url 'https://repo.gradle.org/gradle/libs-releases' }
jcenter()
maven { url "https://plugins.gradle.org/m2/" }
}
dependencies {
// Jar
classpath "com.github.jengelman.gradle.plugins:shadow:6.0.0"
}
}
plugins {
// Apply the java plugin to add support for Java
id 'java'
// Apply the application plugin to add support for building an application
id 'application'
}
apply plugin: "com.github.johnrengelman.shadow"
shadowJar {
baseName = 'iRODSKubernetes'
classifier = null
version = null
configurations = [project.configurations.runtimeClasspath]
}
jar {
manifest {
attributes 'Main-Class': 'nl.wur.ssb.App'
}
}
artifacts {
archives shadowJar
}
group = 'nl.munlock.kubernetes'
version = '0.1.0'
description = """Kubernetes manager for unlock"""
repositories {
// Use jcenter for resolving your dependencies.
// You can declare any Maven/Ivy/file repository here.
jcenter()
mavenLocal()
}
dependencies {
// https://mvnrepository.com/artifact/com.google.guava/guava
compile group: 'com.google.guava', name: 'guava', version: '27.1-jre'
// This dependency is found on compile classpath of this component and consumers.
implementation 'com.google.guava:guava:26.0-jre'
// Use JUnit test framework
testImplementation 'junit:junit:4.12'
// https://mvnrepository.com/artifact/org.apache.poi/poi
compile group: 'org.apache.poi', name: 'poi', version: '4.1.0'
// https://mvnrepository.com/artifact/org.apache.poi/poi-ooxml
compile group: 'org.apache.poi', name: 'poi-ooxml', version: '4.1.0'
// https://mvnrepository.com/artifact/com.github.samtools/htsjdk
compile group: 'com.github.samtools', name: 'htsjdk', version: '2.19.0'
// https://mvnrepository.com/artifact/com.jcraft/jsch
compile group: 'com.jcraft', name: 'jsch', version: '0.1.55'
// Kubernetes api
compile 'io.kubernetes:client-java:5.0.0'
// Logger
compile group: 'log4j', name: 'log4j', version:'1.2.17'
// https://mvnrepository.com/artifact/com.esotericsoftware.yamlbeans/yamlbeans
compile group: 'com.esotericsoftware.yamlbeans', name: 'yamlbeans', version: '1.14'
// locally installed jargon and unlock api
compile group: 'jargon', name: 'core', version: '4.3.0.2'
compile group: 'nl.munlock', name: 'unlockapi', version: '1.0.1'
}
// Define the main class for the application
mainClassName = 'nl.wur.ssb.App'
\ No newline at end of file
#!/bin/bash
#============================================================================
#title :IRODS CWL
#description :IRODS manager installation script
#author :Jasper Koehorst
#date :2019
#version :0.0.1
#============================================================================
DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
git -C $DIR pull
# ////////////////////////////////////////////////////////////////////////////////////
# Jargon dependency
# ////////////////////////////////////////////////////////////////////////////////////
wget -nc https://github.com/DICE-UNC/jargon/releases/download/4.3.0.2-RELEASE/jargon-core-4.3.0.2-RELEASE-jar-with-dependencies.jar -O $DIR/jargon-core-4.3.0.2-RELEASE-jar-with-dependencies.jar
mvn install:install-file -Dfile=$DIR/jargon-core-4.3.0.2-RELEASE-jar-with-dependencies.jar -DgroupId=jargon -DartifactId=core -Dversion=4.3.0.2 -Dpackaging=jar
# ////////////////////////////////////////////////////////////////////////////////////
# // UNLOCK API
# ////////////////////////////////////////////////////////////////////////////////////
wget -nc http://download.systemsbiology.nl/unlock/UnlockOntology.jar -O $DIR/UnlockOntology.jar
mvn install:install-file -Dfile=$DIR/UnlockOntology.jar -DgroupId=nl.munlock -DartifactId=unlockapi -Dversion=1.0.1 -Dpackaging=jar
# Building mode
if [ "$1" == "test" ]; then
gradle build -b "$DIR/build.gradle" --info
else
echo "Skipping tests, run './install.sh test' to perform tests"
gradle build -b "$DIR/build.gradle" -x test
fi
cp $DIR/build/libs/*jar $DIR/
# java -jar $DIR/IRODSTransfer.jar --help
/*
* This file was generated by the Gradle 'init' task.
*
* The settings file is used to specify which projects to include in your build.
*
* Detailed information about configuring a multi-project build in Gradle can be found
* in the user guide at https://docs.gradle.org/4.5.1/userguide/multi_project_builds.html
*/
rootProject.name = 'iRODSKubernetes'
package nl.munlock;
import nl.munlock.yaml.Yaml;
import org.apache.log4j.Logger;
import java.util.Arrays;
public class App {
/**
* Main landing point for yaml or kubernetes runs...
* @param args
* @throws Exception
*/
private static final Logger log = Generic.getLogger(Yaml.class, false);
public static void main(String[] args) throws Exception {
if (Arrays.asList(args).contains("-cwl")) {
log.info("Generating YAML files");
Yaml.main(args);
} else if (Arrays.asList(args).contains("-kubernetes")) {
log.info("Executing kubernetes workflows");
nl.munlock.kubernetes.App.main(args);
}
}
}
package nl.munlock;
import nl.munlock.irods.Connection;
import org.apache.commons.io.FilenameUtils;
import org.apache.log4j.*;
import org.irods.jargon.core.exception.JargonException;
import org.irods.jargon.core.pub.DataTransferOperations;
import org.irods.jargon.core.pub.io.IRODSFile;
import org.slf4j.LoggerFactory;
import java.io.File;
import java.io.FileNotFoundException;
import java.util.Scanner;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
public class Generic {
private static final org.slf4j.Logger log = LoggerFactory.getLogger(Generic.class);
/**
* Logger initialization with debug option
* @param debug boolean if debug mode should be enabled
*
* @return
*/
public static Logger getLogger(Class clazz, boolean debug) {
ConsoleAppender console = new ConsoleAppender();
String PATTERN = "%d %-5p [%c{1}] %m%n";
console.setLayout(new PatternLayout(PATTERN));
console.setThreshold(Level.DEBUG);
console.activateOptions();
Logger.getRootLogger().removeAllAppenders();
Logger.getRootLogger().addAppender(console);
Logger logger = Logger.getLogger(clazz);
logger.setLevel(Level.INFO);
if (debug)
logger.setLevel(Level.DEBUG);
FileAppender fa = new FileAppender();
fa.setName("iRODS Logger");
fa.setFile("runner.log");
fa.setLayout(new PatternLayout("%d %-5p [%c{1}] %m%n"));
fa.setThreshold(Level.INFO);
if (debug) fa.setThreshold(Level.DEBUG);
fa.setAppend(true);
fa.activateOptions();
org.apache.log4j.Logger.getRootLogger().addAppender(fa);
return logger;
}
public static String getAssayPath(File collection) throws FileNotFoundException {
String regex = "(.*/S_.*?/(Amplicon|RNA|DNA)/A_.*?)/";
Pattern pattern = Pattern.compile(regex);
Matcher matcher = pattern.matcher(collection.getAbsolutePath() + "/");
while (matcher.find()) {
return matcher.group(1);
}
throw new FileNotFoundException("Assay path not found in " + collection);
}
public static void downloadFile(Connection connection, File download) throws JargonException {
IRODSFile irodsFile = connection.fileFactory.instanceIRODSFile(download.getAbsolutePath());
if (!irodsFile.exists()) {
throw new JargonException("File " + irodsFile + " does not exist");
}
org.apache.log4j.Logger.getLogger("org.nl.wur.ssb.irods.jargon.core.transfer").setLevel(Level.OFF);
DataTransferOperations dataTransferOperationsAO = connection.irodsFileSystem.getIRODSAccessObjectFactory().getDataTransferOperations(connection.irodsAccount);
// Create folder directory
File directory = new File("." + download.getAbsolutePath().replaceAll(download.getName() + "$", ""));
directory.mkdirs();
File localFile = new File("." + download);
while (localFile.exists()) {
localFile.delete();
}
// Disables the logger for the transfer as it easily gives thousands of lines...
org.apache.log4j.Logger.getLogger("org.nl.wur.ssb.irods.jargon.core.transfer").setLevel(Level.OFF);
log.info("Downloading " + localFile.getName());
dataTransferOperationsAO.getOperation(irodsFile, localFile, null, null);
}
public static void destinationChecker(Connection connection, File file) {
Scanner scanner = new Scanner(file.getAbsolutePath());
while (scanner.hasNextLine()) {
String line = scanner.nextLine();
System.err.println(line);
if (line.startsWith("destination")) {
System.err.println(line);
}
}
}
public static String getEdamFormat(String fileName) throws Exception {
// Always remove compression end when gz
fileName = fileName.replaceAll(".gz$","");
String extension = FilenameUtils.getExtension(fileName);
log.info("File extension " + extension + " detected");
if (fileName.startsWith("http")) {
return fileName;
} else if (extension.matches("bam")) {
return "http://edamontology.org/format_2572";
} else if (extension.matches("(fq|fastq)")) {
return "http://edamontology.org/format_1930";
} else if (extension.matches("(fa|fasta)")) {
return "http://edamontology.org/format_1929";
} else {
log.error("Unknown file format " + extension);
return null;
}
}
}
package nl.munlock.irods;
import nl.munlock.options.irods.CommandOptionsIRODS;
import nl.munlock.options.kubernetes.CommandOptionsKubernetes;
import org.irods.jargon.core.connection.ClientServerNegotiationPolicy;
import org.irods.jargon.core.connection.IRODSAccount;
import org.irods.jargon.core.exception.JargonException;
import org.irods.jargon.core.pub.IRODSAccessObjectFactory;
import org.irods.jargon.core.pub.IRODSFileSystem;
import org.irods.jargon.core.pub.io.IRODSFileFactory;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import static org.irods.jargon.core.connection.ClientServerNegotiationPolicy.SslNegotiationPolicy.CS_NEG_REFUSE;
import static org.irods.jargon.core.connection.ClientServerNegotiationPolicy.SslNegotiationPolicy.CS_NEG_REQUIRE;
public class Connection {
public final IRODSFileSystem irodsFileSystem;
public final IRODSAccount irodsAccount;
public final IRODSAccessObjectFactory accessObjectFactory;
public final IRODSFileFactory fileFactory;
private static final Logger log = LoggerFactory.getLogger(Connection.class);
public Connection(CommandOptionsIRODS commandOptions) throws JargonException {
// Initialize account object
irodsAccount = IRODSAccount.instance(commandOptions.host,Integer.parseInt(commandOptions.port),commandOptions.username, commandOptions.password,"",commandOptions.zone,"");
// set SSL settings
ClientServerNegotiationPolicy clientServerNegotiationPolicy = new ClientServerNegotiationPolicy();
ClientServerNegotiationPolicy.SslNegotiationPolicy sslPolicy;
if (commandOptions.sslPolicyString.matches("CS_NEG_REQUIRE")) {
sslPolicy = CS_NEG_REQUIRE;
} else if (commandOptions.sslPolicyString.matches("CS_NEG_REFUSE")) {
sslPolicy = CS_NEG_REFUSE;
} else {
throw new JargonException("SSL policy not recognised: " + commandOptions.sslPolicyString);
}
clientServerNegotiationPolicy.setSslNegotiationPolicy(sslPolicy);
irodsAccount.setClientServerNegotiationPolicy(clientServerNegotiationPolicy);
irodsFileSystem = IRODSFileSystem.instance();
accessObjectFactory = irodsFileSystem.getIRODSAccessObjectFactory();
// AuthResponse x = accessObjectFactory.authenticateIRODSAccount(irodsAccount);
// log.info(x.getAuthMessage());
fileFactory = accessObjectFactory.getIRODSFileFactory(irodsAccount);
}
}
This diff is collapsed.
package nl.munlock.kubernetes;
import nl.munlock.options.kubernetes.CommandOptionsKubernetes;
import nl.munlock.irods.Connection;
import org.apache.log4j.Level;
public class App {
public static void main(String[] args) throws Exception {
org.apache.log4j.Logger.getLogger("org").setLevel(Level.OFF);
CommandOptionsKubernetes commandOptions = new CommandOptionsKubernetes(args);
Connection connection = new Connection(commandOptions);
Kubernetes.main(commandOptions, connection);
}
}
package nl.munlock.kubernetes;
import com.esotericsoftware.yamlbeans.YamlException;
import com.esotericsoftware.yamlbeans.YamlReader;
import com.google.gson.JsonSyntaxException;
import io.kubernetes.client.ApiClient;
import io.kubernetes.client.ApiException;
import io.kubernetes.client.Configuration;
import io.kubernetes.client.apis.BatchV1Api;
import io.kubernetes.client.apis.CoreV1Api;
import io.kubernetes.client.custom.Quantity;
import io.kubernetes.client.models.*;
import io.kubernetes.client.util.Config;
import nl.munlock.Generic;
import nl.munlock.irods.Connection;
import nl.munlock.irods.Search;
import nl.munlock.options.kubernetes.CommandOptionsKubernetes;
import nl.munlock.objects.Workflow;
import org.apache.log4j.Logger;
import org.irods.jargon.core.exception.JargonException;
import org.irods.jargon.core.query.*;
import java.io.File;
import java.io.FileNotFoundException;
import java.io.FileReader;
import java.io.IOException;
import java.util.*;
import java.util.concurrent.TimeUnit;
import static nl.munlock.yaml.Workflow.fixClass;
public class Kubernetes {
private static final Logger log = Generic.getLogger(Kubernetes.class, false);
public static HashSet<String> yamls = new HashSet<>();
public static void main(CommandOptionsKubernetes commandOptionsKubernetes, Connection connection) throws Exception {
getYamls(commandOptionsKubernetes, connection);
createJobs(commandOptionsKubernetes, connection);
}
public static void createJobs(CommandOptionsKubernetes commandOptionsKubernetes, Connection connection) throws IOException, ApiException, JargonQueryException, JargonException, InterruptedException {
// Requires the kube config file
ApiClient client = Config.defaultClient();
client.setDebugging(false);
Configuration.setDefaultApiClient(client);
int assayCount = 0;
// Check nl.wur.ssb.kubernetes if it has room for more jobs...
int totalItems = checkKubernetesJobs(client);
for (String yaml : yamls) {
assayCount = assayCount + 1;
V1Job v1Job = generateKubernetesJobObject(commandOptionsKubernetes, yaml, connection);
// Sleeping for 1 minute...
while (totalItems >= commandOptionsKubernetes.limit) {
log.info("Sleeping for 1 minute as there are " + totalItems + " jobs running");
// Count down?
for (int i = 60; i >= 0; i--) {
System.out.print("Sleeping for " + i + " seconds\r");
TimeUnit.SECONDS.sleep(1);
}
totalItems = checkKubernetesJobs(client);
}
// v1Job.setKind("my-kind");
BatchV1Api batchV1Api = new BatchV1Api();
// Submit the job to nl.wur.ssb.kubernetes
batchV1Api.createNamespacedJob("unlock", v1Job, false, null, null);
batchV1Api.createNamespacedJobCall("unlock", v1Job, null, null, null, null, null);
TimeUnit.MILLISECONDS.sleep(500);
totalItems = totalItems + 1;
int remain = yamls.size() - assayCount;
log.info("Jobs currently running " + totalItems + " of " + remain);
}
yamls = new HashSet<>();
}
private static void getYamls(CommandOptionsKubernetes commandOptionsKubernetes, Connection connection) throws JargonException, GenQueryBuilderException, JargonQueryException, IOException, ApiException, InterruptedException {
// List all project turtle files...
Search.getAllUnprocessed(commandOptionsKubernetes, connection);
Kubernetes.createJobs(commandOptionsKubernetes, connection);
Search.getAllUnprocessedReferences(commandOptionsKubernetes, connection);
Kubernetes.createJobs(commandOptionsKubernetes, connection);
Search.getAllYamls(commandOptionsKubernetes, connection);
Kubernetes.createJobs(commandOptionsKubernetes, connection);
}
/**
* Only manages the irodsrunner- jobs... and cleans them up when needed...
*
* @return
* @throws ApiException
* @param client
*/
public static int checkKubernetesJobs(ApiClient client) throws ApiException {
BatchV1Api batchV1Api = new BatchV1Api(client);
List<V1Job> v1Jobs = batchV1Api.listNamespacedJob("unlock", null, null, null, null, null, null, null, null, null).getItems();
int totalJobs = v1Jobs.size();
log.info("Total jobs before removal " + totalJobs);
Iterator<V1Job> v1JobIterator = v1Jobs.iterator();
while (v1JobIterator.hasNext()) {
V1Job v1Job = v1JobIterator.next();
if (v1Job.getMetadata().getName().startsWith("cwl-")) {
if (v1Job.getStatus().getActive() == null) {
log.info("Deleting " + v1Job.getMetadata().getName() + " " + totalJobs);
try {
batchV1Api.deleteNamespacedJob(v1Job.getMetadata().getName(), "unlock", null, null, null, null, null, null);
} catch (com.google.gson.JsonSyntaxException | io.kubernetes.client.ApiException e) {
if (e.getMessage().contains("Expected a string but was BEGIN_OBJECT")) {
// Exception thrown hen item is being deleted, api is waiting for new implementation...
} else if (e.getMessage().contains("Not Found")) {
// Job was already on the way to deletion and we went to fast?
} else {
throw new JsonSyntaxException(e.getMessage());
}
totalJobs = totalJobs - 1;
}
}
}
}
// Remove pods as TTL is not always working when finished
CoreV1Api api = new CoreV1Api();
V1PodList v1PodList = api.listPodForAllNamespaces(null, null, null, null, null, null, null, null, null);
Iterator<V1Pod> v1PodIterator = v1PodList.getItems().iterator();
while (v1PodIterator.hasNext()) {
V1Pod v1Pod = v1PodIterator.next();
if (v1Pod.getMetadata().getName().startsWith("cwl-")) {
if (v1Pod.getStatus().getPhase().matches("Succeeded")) {
log.info("Deleting " + v1Pod.getMetadata().getName() + " " + totalJobs);
try {
api.deleteNamespacedPod(v1Pod.getMetadata().getName(), v1Pod.getMetadata().getNamespace(), null, null, null, null, null, null);
} catch (com.google.gson.JsonSyntaxException | io.kubernetes.client.ApiException e) {
if (e.getMessage().contains("Expected a string but was BEGIN_OBJECT")) {
// Exception thrown hen item is being deleted, api is waiting for new implementation...
} else if (e.getMessage().contains("Not Found")) {
// Job was already on the way to deletion and we went to fast?
} else {
throw new JsonSyntaxException(e.getMessage());
}
totalJobs = totalJobs - 1;
}
}
}
}
if (totalJobs < 0) {
totalJobs = 0;
}
log.info("Jobs currently running " + totalJobs);
return totalJobs;
}
private static V1Job generateKubernetesJobObject(CommandOptionsKubernetes commandOptions, String yamlFile, Connection connection) throws JargonException, JargonQueryException, YamlException, FileNotFoundException {
// Load yaml file for cores and memory restrictions?
log.info("Loading ." + yamlFile);
fixClass("." + yamlFile);
YamlReader reader = new YamlReader(new FileReader("." + yamlFile));
// Hack as ngtax currently cannot handle the -!nl.... paths in the yaml file
Workflow workflow;
try {
workflow = (Workflow) reader.read();
} catch (YamlReader.YamlReaderException e) {
Scanner scanner = new Scanner(new File("." + yamlFile));
workflow = new Workflow();
while (scanner.hasNextLine()) {
String line = scanner.nextLine();
if (line.startsWith("threads")) {
workflow.threads = Integer.parseInt(line.split(" ")[1]);
}
if (line.startsWith("memory")) {
workflow.memory = Integer.parseInt(line.split(" ")[1]);
}
}
}
if (workflow.threads == 0) {
workflow.threads = 2;
}