Skip to content
GitLab
Commit 1b4a53fe authored by Jasper Koehorst's avatar Jasper Koehorst
Browse files

download improvements for preserve

parent d5113fdc
Pipeline #44357 passed with stage
in 1 minute and 18 seconds
Hide whitespace changes
Inline Side-by-side
src/main/java/nl/wur/ssb/Download.java
View file @ 1b4a53fe
......@@ -53,6 +53,7 @@ public class Download {
// Perform a query?
HashSet<String> irodsFiles = IQuest.getFiles(connection, collection);
for (String irodsFile : irodsFiles) {
System.err.println("IRODS FILE: " + irodsFile);
Download.downloadFile(commandOptions, connection, irodsFile);
}
//
......@@ -82,84 +83,90 @@ public class Download {
// org.apache.log4j.Logger.getLogger("org.irods.jargon.core.transfer").setLevel(Level.OFF);
DataTransferOperations dataTransferOperationsAO = connection.irodsFileSystem.getIRODSAccessObjectFactory().getDataTransferOperations(connection.irodsAccount);
// When local directory is a folder then the file should be stored within
File localFile = commandOptions.localDirectory;
// Set file to local directory
File localFile = new File(commandOptions.localDirectory +"/"+ new File(irodsFilePath).getName());
if (commandOptions.preserve) {
// Add path of folder to localDirectory
localFile = new File(commandOptions.localDirectory.getAbsolutePath() + "/" + irodsFile.getAbsolutePath());
// Add path of folder to localDirectory,
String newPath = commandOptions.localDirectory.getAbsolutePath() + "/" + irodsFile.getAbsolutePath();
newPath = newPath.replaceAll("/+", "/");
localFile = new File(newPath);
}
// If the file or folder exists
if (localFile.exists()) {
// If the location points to a file
if (localFile.isFile()) {
// Overwrite when force flag is available
if (commandOptions.force) {
// Get local HASH
LocalChecksumComputerFactory factory = new LocalChecksumComputerFactoryImpl();
SHA256LocalChecksumComputerStrategy localActual = (SHA256LocalChecksumComputerStrategy) factory.instance(ChecksumEncodingEnum.SHA256);
ChecksumValue localChecksumValue = localActual.computeChecksumValueForLocalFile(localFile.getAbsolutePath());
// Get remote hash
DataObjectChecksumUtilitiesAO dataObjectChecksumUtilitiesAO = connection.irodsFileSystem.getIRODSAccessObjectFactory().getDataObjectChecksumUtilitiesAO(connection.irodsAccount);
ChecksumValue remoteChecksumValue = dataObjectChecksumUtilitiesAO.computeChecksumOnDataObject(irodsFile);
if (localChecksumValue.getBase64ChecksumValue().contains(remoteChecksumValue.getBase64ChecksumValue())) {
logger.info("Same checksum, not going to overwrite");
return;
} else {
logger.info("Remove local file location " + localFile + " " + localChecksumValue.getBase64ChecksumValue());
logger.info("Does not match checksum of " + irodsFile.getAbsolutePath() + " " + remoteChecksumValue.getBase64ChecksumValue());
localFile.delete();
}
} else {
throw new FileAlreadyExistsException("File already exists use --force to overwrite the file: " + localFile);
}
}
// If it is a directory
if (commandOptions.localDirectory.isDirectory()) {
localFile = new File(commandOptions.localDirectory + "/" + new File(irodsFilePath).getName());
if (localFile.exists()) {
if (localFile.isFile()) {
if (commandOptions.force) {
// If bigger than 50gb... do a byte size check?
if (localFile.length() / (1024 * 1024 * 1024) > 50) {
if (localFile.length() != irodsFile.length()) {
// Delete local file
while (localFile.exists()) {
localFile.delete();
}
} else {
logger.info("File is big thus skipping checksum. Byte size is the same");
}
} else {
// Get local HASH
logger.info("Computing local checksum of " + localFile);
LocalChecksumComputerFactory factory = new LocalChecksumComputerFactoryImpl();
SHA256LocalChecksumComputerStrategy localActual = (SHA256LocalChecksumComputerStrategy) factory.instance(ChecksumEncodingEnum.SHA256);
ChecksumValue localChecksumValue = localActual.computeChecksumValueForLocalFile(localFile.getAbsolutePath());
// Get remote hash
DataObjectChecksumUtilitiesAO dataObjectChecksumUtilitiesAO = connection.irodsFileSystem.getIRODSAccessObjectFactory().getDataObjectChecksumUtilitiesAO(connection.irodsAccount);
ChecksumValue remoteChecksumValue = dataObjectChecksumUtilitiesAO.computeChecksumOnDataObject(irodsFile);
if (localChecksumValue.getBase64ChecksumValue().contains(remoteChecksumValue.getBase64ChecksumValue())) {
logger.info("Same checksum, not going to overwrite");
return;
} else {
logger.info("Remove local file location " + localFile + " " + localChecksumValue.getBase64ChecksumValue());
logger.info("Does not match checksum of " + irodsFile.getAbsolutePath() + " " + remoteChecksumValue.getBase64ChecksumValue());
localFile.delete();
}
}
} else {
throw new FileAlreadyExistsException("File already exists use --force to overwrite the file: " + localFile);
}
// Perform a simple file size check by default
if (localFile.length() != irodsFile.length()){
while (localFile.exists()) {
logger.error("File size does not match, scheduled for downloading");
localFile.delete();
}
if (localFile.isDirectory()) {
throw new FileAlreadyExistsException("Folder instead of a file was detected at " + localFile);
} else if (commandOptions.force) {
// Get local HASH
LocalChecksumComputerFactory factory = new LocalChecksumComputerFactoryImpl();
SHA256LocalChecksumComputerStrategy localActual = (SHA256LocalChecksumComputerStrategy) factory.instance(ChecksumEncodingEnum.SHA256);
ChecksumValue localChecksumValue = localActual.computeChecksumValueForLocalFile(localFile.getAbsolutePath());
// Get remote hash
DataObjectChecksumUtilitiesAO dataObjectChecksumUtilitiesAO = connection.irodsFileSystem.getIRODSAccessObjectFactory().getDataObjectChecksumUtilitiesAO(connection.irodsAccount);
ChecksumValue remoteChecksumValue = dataObjectChecksumUtilitiesAO.computeChecksumOnDataObject(irodsFile);
if (localChecksumValue.getBase64ChecksumValue().contains(remoteChecksumValue.getBase64ChecksumValue())) {
logger.info("Same checksum, not going to overwrite");
return;
} else {
logger.info("Remove local file location " + localFile + " " + localChecksumValue.getBase64ChecksumValue());
logger.info("Does not match checksum of " + irodsFile.getAbsolutePath() + " " + remoteChecksumValue.getBase64ChecksumValue());
localFile.delete();
}
} else {
logger.info("File already exists use --force to overwrite the file: " + localFile);
}
}
}
// if (commandOptions.localDirectory.isDirectory()) {
// if (localFile.exists()) {
// if (localFile.isFile()) {
// if (commandOptions.force) {
// // If bigger than 50gb... do a byte size check?
// if (localFile.length() / (1024 * 1024 * 1024) > 50) {
// if (localFile.length() != irodsFile.length()) {
// // Delete local file
// while (localFile.exists()) {
// localFile.delete();
// }
// } else {
// logger.info("File is big thus skipping checksum. Byte size is the same");
// }
// } else {
// // Get local HASH
// logger.info("Computing local checksum of " + localFile);
// LocalChecksumComputerFactory factory = new LocalChecksumComputerFactoryImpl();
// SHA256LocalChecksumComputerStrategy localActual = (SHA256LocalChecksumComputerStrategy) factory.instance(ChecksumEncodingEnum.SHA256);
// ChecksumValue localChecksumValue = localActual.computeChecksumValueForLocalFile(localFile.getAbsolutePath());
// // Get remote hash
// DataObjectChecksumUtilitiesAO dataObjectChecksumUtilitiesAO = connection.irodsFileSystem.getIRODSAccessObjectFactory().getDataObjectChecksumUtilitiesAO(connection.irodsAccount);
// ChecksumValue remoteChecksumValue = dataObjectChecksumUtilitiesAO.computeChecksumOnDataObject(irodsFile);
// if (localChecksumValue.getBase64ChecksumValue().contains(remoteChecksumValue.getBase64ChecksumValue())) {
// logger.info("Same checksum, not going to overwrite");
// return;
// } else {
// logger.info("Remove local file location " + localFile + " " + localChecksumValue.getBase64ChecksumValue());
// logger.info("Does not match checksum of " + irodsFile.getAbsolutePath() + " " + remoteChecksumValue.getBase64ChecksumValue());
// localFile.delete();
// }
// }
// } else {
// logger.info("File already exists use --force to overwrite the file: " + localFile);
// }
// }
// if (localFile.isDirectory()) {
// throw new Error("Folder instead of a file was detected at " + localFile);
// }
// }
// }
// }
// THE ACTUAL TRANSFER
// Disables the logger for the transfer as it easily gives thousands of lines... and perform the transfer
......@@ -172,7 +179,6 @@ public class Download {
logger.info("Downloading file from " + irodsFile + " to " + localFile.getAbsolutePath());
try {
StatusCallbackListener statusCallbackListener = new StatusCallbackListener();
TransferControlBlock defaultTransferControlBlock = DefaultTransferControlBlock.instance();
TransferOptions transferOptions = dataTransferOperationsAO.buildTransferOptionsBasedOnJargonProperties();
transferOptions.setIntraFileStatusCallbacks(true);
......
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment