Skip to content
GitLab
Menu
Projects
Groups
Snippets
Help
Help
Support
Community forum
Keyboard shortcuts
?
Submit feedback
Contribute to GitLab
Sign in
Toggle navigation
Menu
Open sidebar
UNLOCK
kubernetes
Commits
ebb4159e
Commit
ebb4159e
authored
Oct 29, 2020
by
Koehorst, Jasper
Browse files
Initial commit
parents
Pipeline
#14533
failed with stage
Changes
45
Pipelines
1
Expand all
Hide whitespace changes
Inline
Side-by-side
.gitlab-ci.yml
0 → 100644
View file @
ebb4159e
image
:
name
:
munlock/build
entrypoint
:
[
"
"
]
# force an empty entrypoint
#stages to execute
stages
:
-
test_build
before_script
:
-
apt-get update
-
apt-get install -y lftp
-
export PATH="/root/.sdkman/candidates/maven/current/bin:/root/.sdkman/candidates/java/current/bin:/root/.sdkman/candidates/gradle/current/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"
#test the gradle build
test_build_dev
:
stage
:
test_build
script
:
-
echo $0
-
echo $PATH
-
./build.sh
-
lftp sftp://$USERNAME:$PASSWORD@$HOST -e "set sftp:auto-confirm yes; cd upload/unlock/;put ./iRODSKubernetes.jar; bye"
only
:
-
dev
#test the gradle build
test_build_master
:
stage
:
test_build
script
:
-
echo $0
-
echo $PATH
-
./build.sh
-
lftp sftp://$USERNAME:$PASSWORD@$HOST -e "set sftp:auto-confirm yes; cd upload/unlock/;put ./iRODSKubernetes.jar; bye"
only
:
-
master
\ No newline at end of file
build.gradle
0 → 100644
View file @
ebb4159e
/*
* This file was generated by the Gradle 'init' task.
*
* This generated file contains a sample Java project to get you started.
* For more details take a look at the Java Quickstart chapter in the Gradle
* user guide available at https://docs.gradle.org/5.0/userguide/tutorial_java_projects.html
*/
buildscript
{
repositories
{
mavenCentral
()
mavenLocal
()
maven
{
url
'https://repo.gradle.org/gradle/libs-releases'
}
jcenter
()
maven
{
url
"https://plugins.gradle.org/m2/"
}
}
dependencies
{
// Jar
classpath
"com.github.jengelman.gradle.plugins:shadow:6.0.0"
}
}
plugins
{
// Apply the java plugin to add support for Java
id
'java'
// Apply the application plugin to add support for building an application
id
'application'
}
apply
plugin:
"com.github.johnrengelman.shadow"
shadowJar
{
baseName
=
'iRODSKubernetes'
classifier
=
null
version
=
null
configurations
=
[
project
.
configurations
.
runtimeClasspath
]
}
jar
{
manifest
{
attributes
'Main-Class'
:
'nl.wur.ssb.App'
}
}
artifacts
{
archives
shadowJar
}
group
=
'nl.munlock.kubernetes'
version
=
'0.1.0'
description
=
"""Kubernetes manager for unlock"""
repositories
{
// Use jcenter for resolving your dependencies.
// You can declare any Maven/Ivy/file repository here.
jcenter
()
mavenLocal
()
}
dependencies
{
// https://mvnrepository.com/artifact/com.google.guava/guava
compile
group:
'com.google.guava'
,
name:
'guava'
,
version:
'27.1-jre'
// This dependency is found on compile classpath of this component and consumers.
implementation
'com.google.guava:guava:26.0-jre'
// Use JUnit test framework
testImplementation
'junit:junit:4.12'
// https://mvnrepository.com/artifact/org.apache.poi/poi
compile
group:
'org.apache.poi'
,
name:
'poi'
,
version:
'4.1.0'
// https://mvnrepository.com/artifact/org.apache.poi/poi-ooxml
compile
group:
'org.apache.poi'
,
name:
'poi-ooxml'
,
version:
'4.1.0'
// https://mvnrepository.com/artifact/com.github.samtools/htsjdk
compile
group:
'com.github.samtools'
,
name:
'htsjdk'
,
version:
'2.19.0'
// https://mvnrepository.com/artifact/com.jcraft/jsch
compile
group:
'com.jcraft'
,
name:
'jsch'
,
version:
'0.1.55'
// Kubernetes api
compile
'io.kubernetes:client-java:5.0.0'
// Logger
compile
group:
'log4j'
,
name:
'log4j'
,
version:
'1.2.17'
// https://mvnrepository.com/artifact/com.esotericsoftware.yamlbeans/yamlbeans
compile
group:
'com.esotericsoftware.yamlbeans'
,
name:
'yamlbeans'
,
version:
'1.14'
// locally installed jargon and unlock api
compile
group:
'jargon'
,
name:
'core'
,
version:
'4.3.0.2'
compile
group:
'nl.munlock'
,
name:
'unlockapi'
,
version:
'1.0.1'
}
// Define the main class for the application
mainClassName
=
'nl.wur.ssb.App'
\ No newline at end of file
build.sh
0 → 100755
View file @
ebb4159e
#!/bin/bash
#============================================================================
#title :IRODS CWL
#description :IRODS manager installation script
#author :Jasper Koehorst
#date :2019
#version :0.0.1
#============================================================================
DIR
=
"
$(
cd
"
$(
dirname
"
${
BASH_SOURCE
[0]
}
"
)
"
&&
pwd
)
"
git
-C
$DIR
pull
# ////////////////////////////////////////////////////////////////////////////////////
# Jargon dependency
# ////////////////////////////////////////////////////////////////////////////////////
wget
-nc
https://github.com/DICE-UNC/jargon/releases/download/4.3.0.2-RELEASE/jargon-core-4.3.0.2-RELEASE-jar-with-dependencies.jar
-O
$DIR
/jargon-core-4.3.0.2-RELEASE-jar-with-dependencies.jar
mvn
install
:install-file
-Dfile
=
$DIR
/jargon-core-4.3.0.2-RELEASE-jar-with-dependencies.jar
-DgroupId
=
jargon
-DartifactId
=
core
-Dversion
=
4.3.0.2
-Dpackaging
=
jar
# ////////////////////////////////////////////////////////////////////////////////////
# // UNLOCK API
# ////////////////////////////////////////////////////////////////////////////////////
wget
-nc
http://download.systemsbiology.nl/unlock/UnlockOntology.jar
-O
$DIR
/UnlockOntology.jar
mvn
install
:install-file
-Dfile
=
$DIR
/UnlockOntology.jar
-DgroupId
=
nl.munlock
-DartifactId
=
unlockapi
-Dversion
=
1.0.1
-Dpackaging
=
jar
# Building mode
if
[
"
$1
"
==
"test"
]
;
then
gradle build
-b
"
$DIR
/build.gradle"
--info
else
echo
"Skipping tests, run './install.sh test' to perform tests"
gradle build
-b
"
$DIR
/build.gradle"
-x
test
fi
cp
$DIR
/build/libs/
*
jar
$DIR
/
# java -jar $DIR/IRODSTransfer.jar --help
settings.gradle
0 → 100644
View file @
ebb4159e
/*
* This file was generated by the Gradle 'init' task.
*
* The settings file is used to specify which projects to include in your build.
*
* Detailed information about configuring a multi-project build in Gradle can be found
* in the user guide at https://docs.gradle.org/4.5.1/userguide/multi_project_builds.html
*/
rootProject
.
name
=
'iRODSKubernetes'
src/main/java/nl/munlock/App.java
0 → 100644
View file @
ebb4159e
package
nl.munlock
;
import
nl.munlock.yaml.Yaml
;
import
org.apache.log4j.Logger
;
import
java.util.Arrays
;
public
class
App
{
/**
* Main landing point for yaml or kubernetes runs...
* @param args
* @throws Exception
*/
private
static
final
Logger
log
=
Generic
.
getLogger
(
Yaml
.
class
,
false
);
public
static
void
main
(
String
[]
args
)
throws
Exception
{
if
(
Arrays
.
asList
(
args
).
contains
(
"-cwl"
))
{
log
.
info
(
"Generating YAML files"
);
Yaml
.
main
(
args
);
}
else
if
(
Arrays
.
asList
(
args
).
contains
(
"-kubernetes"
))
{
log
.
info
(
"Executing kubernetes workflows"
);
nl
.
munlock
.
kubernetes
.
App
.
main
(
args
);
}
}
}
src/main/java/nl/munlock/Generic.java
0 → 100644
View file @
ebb4159e
package
nl.munlock
;
import
nl.munlock.irods.Connection
;
import
org.apache.commons.io.FilenameUtils
;
import
org.apache.log4j.*
;
import
org.irods.jargon.core.exception.JargonException
;
import
org.irods.jargon.core.pub.DataTransferOperations
;
import
org.irods.jargon.core.pub.io.IRODSFile
;
import
org.slf4j.LoggerFactory
;
import
java.io.File
;
import
java.io.FileNotFoundException
;
import
java.util.Scanner
;
import
java.util.regex.Matcher
;
import
java.util.regex.Pattern
;
public
class
Generic
{
private
static
final
org
.
slf4j
.
Logger
log
=
LoggerFactory
.
getLogger
(
Generic
.
class
);
/**
* Logger initialization with debug option
* @param debug boolean if debug mode should be enabled
*
* @return
*/
public
static
Logger
getLogger
(
Class
clazz
,
boolean
debug
)
{
ConsoleAppender
console
=
new
ConsoleAppender
();
String
PATTERN
=
"%d %-5p [%c{1}] %m%n"
;
console
.
setLayout
(
new
PatternLayout
(
PATTERN
));
console
.
setThreshold
(
Level
.
DEBUG
);
console
.
activateOptions
();
Logger
.
getRootLogger
().
removeAllAppenders
();
Logger
.
getRootLogger
().
addAppender
(
console
);
Logger
logger
=
Logger
.
getLogger
(
clazz
);
logger
.
setLevel
(
Level
.
INFO
);
if
(
debug
)
logger
.
setLevel
(
Level
.
DEBUG
);
FileAppender
fa
=
new
FileAppender
();
fa
.
setName
(
"iRODS Logger"
);
fa
.
setFile
(
"runner.log"
);
fa
.
setLayout
(
new
PatternLayout
(
"%d %-5p [%c{1}] %m%n"
));
fa
.
setThreshold
(
Level
.
INFO
);
if
(
debug
)
fa
.
setThreshold
(
Level
.
DEBUG
);
fa
.
setAppend
(
true
);
fa
.
activateOptions
();
org
.
apache
.
log4j
.
Logger
.
getRootLogger
().
addAppender
(
fa
);
return
logger
;
}
public
static
String
getAssayPath
(
File
collection
)
throws
FileNotFoundException
{
String
regex
=
"(.*/S_.*?/(Amplicon|RNA|DNA)/A_.*?)/"
;
Pattern
pattern
=
Pattern
.
compile
(
regex
);
Matcher
matcher
=
pattern
.
matcher
(
collection
.
getAbsolutePath
()
+
"/"
);
while
(
matcher
.
find
())
{
return
matcher
.
group
(
1
);
}
throw
new
FileNotFoundException
(
"Assay path not found in "
+
collection
);
}
public
static
void
downloadFile
(
Connection
connection
,
File
download
)
throws
JargonException
{
IRODSFile
irodsFile
=
connection
.
fileFactory
.
instanceIRODSFile
(
download
.
getAbsolutePath
());
if
(!
irodsFile
.
exists
())
{
throw
new
JargonException
(
"File "
+
irodsFile
+
" does not exist"
);
}
org
.
apache
.
log4j
.
Logger
.
getLogger
(
"org.nl.wur.ssb.irods.jargon.core.transfer"
).
setLevel
(
Level
.
OFF
);
DataTransferOperations
dataTransferOperationsAO
=
connection
.
irodsFileSystem
.
getIRODSAccessObjectFactory
().
getDataTransferOperations
(
connection
.
irodsAccount
);
// Create folder directory
File
directory
=
new
File
(
"."
+
download
.
getAbsolutePath
().
replaceAll
(
download
.
getName
()
+
"$"
,
""
));
directory
.
mkdirs
();
File
localFile
=
new
File
(
"."
+
download
);
while
(
localFile
.
exists
())
{
localFile
.
delete
();
}
// Disables the logger for the transfer as it easily gives thousands of lines...
org
.
apache
.
log4j
.
Logger
.
getLogger
(
"org.nl.wur.ssb.irods.jargon.core.transfer"
).
setLevel
(
Level
.
OFF
);
log
.
info
(
"Downloading "
+
localFile
.
getName
());
dataTransferOperationsAO
.
getOperation
(
irodsFile
,
localFile
,
null
,
null
);
}
public
static
void
destinationChecker
(
Connection
connection
,
File
file
)
{
Scanner
scanner
=
new
Scanner
(
file
.
getAbsolutePath
());
while
(
scanner
.
hasNextLine
())
{
String
line
=
scanner
.
nextLine
();
System
.
err
.
println
(
line
);
if
(
line
.
startsWith
(
"destination"
))
{
System
.
err
.
println
(
line
);
}
}
}
public
static
String
getEdamFormat
(
String
fileName
)
throws
Exception
{
// Always remove compression end when gz
fileName
=
fileName
.
replaceAll
(
".gz$"
,
""
);
String
extension
=
FilenameUtils
.
getExtension
(
fileName
);
log
.
info
(
"File extension "
+
extension
+
" detected"
);
if
(
fileName
.
startsWith
(
"http"
))
{
return
fileName
;
}
else
if
(
extension
.
matches
(
"bam"
))
{
return
"http://edamontology.org/format_2572"
;
}
else
if
(
extension
.
matches
(
"(fq|fastq)"
))
{
return
"http://edamontology.org/format_1930"
;
}
else
if
(
extension
.
matches
(
"(fa|fasta)"
))
{
return
"http://edamontology.org/format_1929"
;
}
else
{
log
.
error
(
"Unknown file format "
+
extension
);
return
null
;
}
}
}
src/main/java/nl/munlock/irods/Connection.java
0 → 100644
View file @
ebb4159e
package
nl.munlock.irods
;
import
nl.munlock.options.irods.CommandOptionsIRODS
;
import
nl.munlock.options.kubernetes.CommandOptionsKubernetes
;
import
org.irods.jargon.core.connection.ClientServerNegotiationPolicy
;
import
org.irods.jargon.core.connection.IRODSAccount
;
import
org.irods.jargon.core.exception.JargonException
;
import
org.irods.jargon.core.pub.IRODSAccessObjectFactory
;
import
org.irods.jargon.core.pub.IRODSFileSystem
;
import
org.irods.jargon.core.pub.io.IRODSFileFactory
;
import
org.slf4j.Logger
;
import
org.slf4j.LoggerFactory
;
import
static
org
.
irods
.
jargon
.
core
.
connection
.
ClientServerNegotiationPolicy
.
SslNegotiationPolicy
.
CS_NEG_REFUSE
;
import
static
org
.
irods
.
jargon
.
core
.
connection
.
ClientServerNegotiationPolicy
.
SslNegotiationPolicy
.
CS_NEG_REQUIRE
;
public
class
Connection
{
public
final
IRODSFileSystem
irodsFileSystem
;
public
final
IRODSAccount
irodsAccount
;
public
final
IRODSAccessObjectFactory
accessObjectFactory
;
public
final
IRODSFileFactory
fileFactory
;
private
static
final
Logger
log
=
LoggerFactory
.
getLogger
(
Connection
.
class
);
public
Connection
(
CommandOptionsIRODS
commandOptions
)
throws
JargonException
{
// Initialize account object
irodsAccount
=
IRODSAccount
.
instance
(
commandOptions
.
host
,
Integer
.
parseInt
(
commandOptions
.
port
),
commandOptions
.
username
,
commandOptions
.
password
,
""
,
commandOptions
.
zone
,
""
);
// set SSL settings
ClientServerNegotiationPolicy
clientServerNegotiationPolicy
=
new
ClientServerNegotiationPolicy
();
ClientServerNegotiationPolicy
.
SslNegotiationPolicy
sslPolicy
;
if
(
commandOptions
.
sslPolicyString
.
matches
(
"CS_NEG_REQUIRE"
))
{
sslPolicy
=
CS_NEG_REQUIRE
;
}
else
if
(
commandOptions
.
sslPolicyString
.
matches
(
"CS_NEG_REFUSE"
))
{
sslPolicy
=
CS_NEG_REFUSE
;
}
else
{
throw
new
JargonException
(
"SSL policy not recognised: "
+
commandOptions
.
sslPolicyString
);
}
clientServerNegotiationPolicy
.
setSslNegotiationPolicy
(
sslPolicy
);
irodsAccount
.
setClientServerNegotiationPolicy
(
clientServerNegotiationPolicy
);
irodsFileSystem
=
IRODSFileSystem
.
instance
();
accessObjectFactory
=
irodsFileSystem
.
getIRODSAccessObjectFactory
();
// AuthResponse x = accessObjectFactory.authenticateIRODSAccount(irodsAccount);
// log.info(x.getAuthMessage());
fileFactory
=
accessObjectFactory
.
getIRODSFileFactory
(
irodsAccount
);
}
}
src/main/java/nl/munlock/irods/Search.java
0 → 100644
View file @
ebb4159e
This diff is collapsed.
Click to expand it.
src/main/java/nl/munlock/kubernetes/App.java
0 → 100644
View file @
ebb4159e
package
nl.munlock.kubernetes
;
import
nl.munlock.options.kubernetes.CommandOptionsKubernetes
;
import
nl.munlock.irods.Connection
;
import
org.apache.log4j.Level
;
public
class
App
{
public
static
void
main
(
String
[]
args
)
throws
Exception
{
org
.
apache
.
log4j
.
Logger
.
getLogger
(
"org"
).
setLevel
(
Level
.
OFF
);
CommandOptionsKubernetes
commandOptions
=
new
CommandOptionsKubernetes
(
args
);
Connection
connection
=
new
Connection
(
commandOptions
);
Kubernetes
.
main
(
commandOptions
,
connection
);
}
}
src/main/java/nl/munlock/kubernetes/Kubernetes.java
0 → 100644
View file @
ebb4159e
package
nl.munlock.kubernetes
;
import
com.esotericsoftware.yamlbeans.YamlException
;
import
com.esotericsoftware.yamlbeans.YamlReader
;
import
com.google.gson.JsonSyntaxException
;
import
io.kubernetes.client.ApiClient
;
import
io.kubernetes.client.ApiException
;
import
io.kubernetes.client.Configuration
;
import
io.kubernetes.client.apis.BatchV1Api
;
import
io.kubernetes.client.apis.CoreV1Api
;
import
io.kubernetes.client.custom.Quantity
;
import
io.kubernetes.client.models.*
;
import
io.kubernetes.client.util.Config
;
import
nl.munlock.Generic
;
import
nl.munlock.irods.Connection
;
import
nl.munlock.irods.Search
;
import
nl.munlock.options.kubernetes.CommandOptionsKubernetes
;
import
nl.munlock.objects.Workflow
;
import
org.apache.log4j.Logger
;
import
org.irods.jargon.core.exception.JargonException
;
import
org.irods.jargon.core.query.*
;
import
java.io.File
;
import
java.io.FileNotFoundException
;
import
java.io.FileReader
;
import
java.io.IOException
;
import
java.util.*
;
import
java.util.concurrent.TimeUnit
;
import
static
nl
.
munlock
.
yaml
.
Workflow
.
fixClass
;
public
class
Kubernetes
{
private
static
final
Logger
log
=
Generic
.
getLogger
(
Kubernetes
.
class
,
false
);
public
static
HashSet
<
String
>
yamls
=
new
HashSet
<>();
public
static
void
main
(
CommandOptionsKubernetes
commandOptionsKubernetes
,
Connection
connection
)
throws
Exception
{
getYamls
(
commandOptionsKubernetes
,
connection
);
createJobs
(
commandOptionsKubernetes
,
connection
);
}
public
static
void
createJobs
(
CommandOptionsKubernetes
commandOptionsKubernetes
,
Connection
connection
)
throws
IOException
,
ApiException
,
JargonQueryException
,
JargonException
,
InterruptedException
{
// Requires the kube config file
ApiClient
client
=
Config
.
defaultClient
();
client
.
setDebugging
(
false
);
Configuration
.
setDefaultApiClient
(
client
);
int
assayCount
=
0
;
// Check nl.wur.ssb.kubernetes if it has room for more jobs...
int
totalItems
=
checkKubernetesJobs
(
client
);
for
(
String
yaml
:
yamls
)
{
assayCount
=
assayCount
+
1
;
V1Job
v1Job
=
generateKubernetesJobObject
(
commandOptionsKubernetes
,
yaml
,
connection
);
// Sleeping for 1 minute...
while
(
totalItems
>=
commandOptionsKubernetes
.
limit
)
{
log
.
info
(
"Sleeping for 1 minute as there are "
+
totalItems
+
" jobs running"
);
// Count down?
for
(
int
i
=
60
;
i
>=
0
;
i
--)
{
System
.
out
.
print
(
"Sleeping for "
+
i
+
" seconds\r"
);
TimeUnit
.
SECONDS
.
sleep
(
1
);
}
totalItems
=
checkKubernetesJobs
(
client
);
}
// v1Job.setKind("my-kind");
BatchV1Api
batchV1Api
=
new
BatchV1Api
();
// Submit the job to nl.wur.ssb.kubernetes
batchV1Api
.
createNamespacedJob
(
"unlock"
,
v1Job
,
false
,
null
,
null
);
batchV1Api
.
createNamespacedJobCall
(
"unlock"
,
v1Job
,
null
,
null
,
null
,
null
,
null
);
TimeUnit
.
MILLISECONDS
.
sleep
(
500
);
totalItems
=
totalItems
+
1
;
int
remain
=
yamls
.
size
()
-
assayCount
;
log
.
info
(
"Jobs currently running "
+
totalItems
+
" of "
+
remain
);
}
yamls
=
new
HashSet
<>();
}
private
static
void
getYamls
(
CommandOptionsKubernetes
commandOptionsKubernetes
,
Connection
connection
)
throws
JargonException
,
GenQueryBuilderException
,
JargonQueryException
,
IOException
,
ApiException
,
InterruptedException
{
// List all project turtle files...
Search
.
getAllUnprocessed
(
commandOptionsKubernetes
,
connection
);
Kubernetes
.
createJobs
(
commandOptionsKubernetes
,
connection
);
Search
.
getAllUnprocessedReferences
(
commandOptionsKubernetes
,
connection
);
Kubernetes
.
createJobs
(
commandOptionsKubernetes
,
connection
);
Search
.
getAllYamls
(
commandOptionsKubernetes
,
connection
);
Kubernetes
.
createJobs
(
commandOptionsKubernetes
,
connection
);
}
/**
* Only manages the irodsrunner- jobs... and cleans them up when needed...
*
* @return
* @throws ApiException
* @param client
*/
public
static
int
checkKubernetesJobs
(
ApiClient
client
)
throws
ApiException
{
BatchV1Api
batchV1Api
=
new
BatchV1Api
(
client
);
List
<
V1Job
>
v1Jobs
=
batchV1Api
.
listNamespacedJob
(
"unlock"
,
null
,
null
,
null
,
null
,
null
,
null
,
null
,
null
,
null
).
getItems
();
int
totalJobs
=
v1Jobs
.
size
();
log
.
info
(
"Total jobs before removal "
+
totalJobs
);
Iterator
<
V1Job
>
v1JobIterator
=
v1Jobs
.
iterator
();
while
(
v1JobIterator
.
hasNext
())
{
V1Job
v1Job
=
v1JobIterator
.
next
();
if
(
v1Job
.
getMetadata
().
getName
().
startsWith
(
"cwl-"
))
{
if
(
v1Job
.
getStatus
().
getActive
()
==
null
)
{
log
.
info
(
"Deleting "
+
v1Job
.
getMetadata
().
getName
()
+
" "
+
totalJobs
);
try
{
batchV1Api
.
deleteNamespacedJob
(
v1Job
.
getMetadata
().
getName
(),
"unlock"
,
null
,
null
,
null
,
null
,
null
,
null
);
}
catch
(
com
.
google
.
gson
.
JsonSyntaxException
|
io
.
kubernetes
.
client
.
ApiException
e
)
{
if
(
e
.
getMessage
().
contains
(
"Expected a string but was BEGIN_OBJECT"
))
{
// Exception thrown hen item is being deleted, api is waiting for new implementation...
}
else
if
(
e
.
getMessage
().
contains
(
"Not Found"
))
{
// Job was already on the way to deletion and we went to fast?
}
else
{
throw
new
JsonSyntaxException
(
e
.
getMessage
());
}
totalJobs
=
totalJobs
-
1
;
}
}
}
}
// Remove pods as TTL is not always working when finished
CoreV1Api
api
=
new
CoreV1Api
();
V1PodList
v1PodList
=
api
.
listPodForAllNamespaces
(
null
,
null
,
null
,
null
,
null
,
null
,
null
,
null
,
null
);
Iterator
<
V1Pod
>
v1PodIterator
=
v1PodList
.
getItems
().
iterator
();
while
(
v1PodIterator
.
hasNext
())
{
V1Pod
v1Pod
=
v1PodIterator
.
next
();
if
(
v1Pod
.
getMetadata
().
getName
().
startsWith
(
"cwl-"
))
{
if
(
v1Pod
.
getStatus
().
getPhase
().
matches
(
"Succeeded"
))
{
log
.
info
(
"Deleting "
+
v1Pod
.
getMetadata
().
getName
()
+
" "
+
totalJobs
);
try
{
api
.
deleteNamespacedPod
(
v1Pod
.
getMetadata
().
getName
(),
v1Pod
.
getMetadata
().
getNamespace
(),
null
,
null
,
null
,
null
,
null
,
null
);
}
catch
(
com
.
google
.
gson
.
JsonSyntaxException
|
io
.
kubernetes
.
client
.
ApiException
e
)
{
if
(
e
.
getMessage
().
contains
(
"Expected a string but was BEGIN_OBJECT"
))
{
// Exception thrown hen item is being deleted, api is waiting for new implementation...
}
else
if
(
e
.
getMessage
().
contains
(
"Not Found"
))
{
// Job was already on the way to deletion and we went to fast?
}
else
{
throw
new
JsonSyntaxException
(
e
.
getMessage
());
}
totalJobs
=
totalJobs
-
1
;
}
}
}
}
if
(
totalJobs
<
0
)
{
totalJobs
=
0
;
}
log
.
info
(
"Jobs currently running "
+
totalJobs
);
return
totalJobs
;
}
private
static
V1Job
generateKubernetesJobObject
(
CommandOptionsKubernetes
commandOptions
,
String
yamlFile
,
Connection
connection
)
throws
JargonException
,
JargonQueryException
,
YamlException
,
FileNotFoundException
{
// Load yaml file for cores and memory restrictions?
log
.
info
(
"Loading ."
+
yamlFile
);
fixClass
(
"."
+
yamlFile
);
YamlReader
reader
=
new
YamlReader
(
new
FileReader
(
"."
+
yamlFile
));
// Hack as ngtax currently cannot handle the -!nl.... paths in the yaml file
Workflow
workflow
;
try
{
workflow
=
(
Workflow
)
reader
.
read
();
}
catch
(
YamlReader
.
YamlReaderException
e
)
{
Scanner
scanner
=
new
Scanner
(
new
File
(
"."
+
yamlFile
));
workflow
=
new
Workflow
();
while
(
scanner
.
hasNextLine
())
{
String
line
=
scanner
.
nextLine
();
if
(
line
.
startsWith
(
"threads"
))
{
workflow
.
threads
=
Integer
.
parseInt
(
line
.
split
(
" "
)[
1
]);
}
if
(
line
.
startsWith
(
"memory"
))
{
workflow
.
memory
=
Integer
.
parseInt
(
line
.
split
(
" "
)[
1
]);
}
}
}
if
(
workflow
.
threads
==
0
)
{