run.py

from joblib import Parallel, delayed
import multiprocessing
import os
import sys
import shlex
import gzip

curl = """curl -X POST -H "Content-Type: application/x-www-form-urlencoded" -d 'result=sample&query=secondary_sample_accession%3D%22XXXXXX%22&format=tsv' "https://www.ebi.ac.uk/ena/portal/api/search\""""
xml = "https://www.ebi.ac.uk/ena/browser/api/xml/"
# wget https://www.ebi.ac.uk/metagenomics/api/v1/genomes/MGYG-HGUT-00001/downloads/MGYG-HGUT-00001.fna
# wget https://www.ebi.ac.uk/metagenomics/api/v1/genomes/MGYG-HGUT-00001/downloads/MGYG-HGUT-00001.gff
# wget https://www.ebi.ac.uk/metagenomics/api/v1/genomes/MGYG-HGUT-00001/downloads/MGYG-HGUT-00001_InterProScan.tsv
# wget https://www.ebi.ac.uk/metagenomics/api/v1/genomes/MGYG-HGUT-02750/downloads/MGYG-HGUT-02750_eggNOG.tsv

source = "https://www.ebi.ac.uk/metagenomics/api/v1/genomes/XXX/downloads/XXX.type"

def retrieval(identifier, accession):
    print("Processing", identifier)
    path = source.replace("XXX", identifier)
    fna = path.replace(".type",".fna")
    gff = path.replace(".type",".gff")
    interpro = path.replace(".type","_InterProScan.tsv")
    egg = path.replace(".type", "_eggNOG.tsv")
    
    # Curl command
    xml_destination = "./genome/" + identifier + "/" + accession + ".xml"
    if not os.path.isfile(xml_destination):
        command = curl.replace("XXXXXX", accession)
        output = run_command(command)
        os.system("wget -nc " + xml + output.split()[2] + " --output-document=" + xml_destination)
        
    # Wget command
    for element in [fna, gff, interpro, egg]:
        # Create the folder
        path = "./genome/" + identifier + "/"
        if not os.path.isdir(path):
            os.makedirs(path)
        # Download the files
        path = "./genome/" + identifier + "/" + element.split("/")[-1]
        if not os.path.isfile(path) and not os.path.isfile(path + ".gz"):
            command = "wget -nc " + element + " --output-document=" + path
            os.system(command)

def run_command(command):
    import subprocess
    result = subprocess.run(shlex.split(command), capture_output=True)
    return result.stdout.decode('ascii')

def conversion(identifier):
    fasta_file = "./genome/" + identifier + "/" + identifier + ".fna"
    gff_file = "./genome/" + identifier + "/" + identifier + ".gff"
    output_file = "./genome/" + identifier + "/" + identifier + ".conversion.ttl"
    
    if os.path.isfile(output_file + ".gz"): return
    
    command("java -jar ./MGnifyParser.jar -tool conversion -f " + fasta_file + " -gff2rdf -i " + gff_file + " -id " + identifier + " -codon 11 -topology linear -o " + output_file)

def interproscan(identifier):
    input_file = "./genome/" + identifier + "/" + identifier + ".conversion.ttl"
    output_file = "./genome/" + identifier + "/" + identifier + ".interproscan.ttl"
    tsv_file = "./genome/" +identifier +"/" + identifier + "_InterProScan.tsv"

    if os.path.isfile(output_file + ".gz"): return

    command("java -jar ./MGnifyParser.jar -tool interpro -i " + input_file + " -o " + output_file + " -tsv " + tsv_file + " -version InterProScan-v5_35-74_0")

def eggnog(identifier):
    input_file = "./genome/" + identifier + "/" + identifier + ".interproscan.ttl"
    tsv_file = "./genome/" +identifier +"/" + identifier + "_eggNOG.tsv"
    output_file = "./genome/" + identifier + "/" + identifier + ".interproscan.eggnog.ttl"
    
    if os.path.isfile(output_file + ".gz"): return

    command("java -jar ./MGnifyParser.jar -tool eggnog -i " + input_file + " -o " + output_file + " -tsv " + tsv_file + " -v v2-db5.0")
    

def gzip(input_file):
    if os.path.isfile(input_file):
        command = "gzip " + input_file
        os.system(command)

def command(command):
    print(command)
    os.system(command)

def conversion_stage(identifier):
    # When eggnog file is present all can be skipped
    output_file = "./genome/" + identifier + "/" + identifier + ".interproscan.eggnog.ttl.gz"
    if os.path.isfile(output_file + ".gz"): return

    conversion(identifier)
    interproscan(identifier)
    eggnog(identifier)

    # Perform gzip compression
    gzip("./genome/" + identifier + "/" + identifier + ".conversion.ttl")
    gzip("./genome/" + identifier + "/" + identifier + ".interproscan.ttl")
    gzip("./genome/" + identifier + "/" + identifier + ".interproscan.eggnog.ttl")


if __name__ == "__main__":
    identifiers = set()
    for line in open("identifiers.tsv"):
        if "MGYG" not in line: continue
        identifier = line.strip().split("\t")[1]
        accession =  line.strip().split("\t")[2]
        if (identifier.startswith("MGYG")):
            retrieval(identifier, accession)
            identifiers.add(identifier)
            conversion_stage(identifier)
            # break
    # Number of cores
    # num_cores = 3 # multiprocessing.cpu_count()
    # results = Parallel(n_jobs=num_cores)(delayed(conversion_stage)(identifier) for identifier in identifiers)