diff --git a/irods/pom.xml b/irods/pom.xml index ba6609fe2c5..9b50155d52e 100644 --- a/irods/pom.xml +++ b/irods/pom.xml @@ -24,6 +24,11 @@ jar + + org.irods + irods4j + 0.5.0-java8 + ch.cyberduck core @@ -37,27 +42,10 @@ ${project.version} - ch.iterate.jargon - jargon-core - 4.2.0.1 - - - com.claymoresystems - puretls - - - org.globus.jglobus - cog-jglobus - - - org.perf4j - perf4j - - - log4j - log4j - - + testcontainers + org.testcontainers + 1.21.3 + test diff --git a/irods/src/main/java/ch/cyberduck/core/irods/DefaultTransferOptionsConfigurer.java b/irods/src/main/java/ch/cyberduck/core/irods/DefaultTransferOptionsConfigurer.java deleted file mode 100644 index f57c2e21217..00000000000 --- a/irods/src/main/java/ch/cyberduck/core/irods/DefaultTransferOptionsConfigurer.java +++ /dev/null @@ -1,38 +0,0 @@ -package ch.cyberduck.core.irods; - -/* - * Copyright (c) 2002-2015 David Kocher. All rights reserved. - * http://cyberduck.ch/ - * - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation; either version 2 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * Bug fixes, suggestions and comments should be sent to feedback@cyberduck.ch - */ - -import ch.cyberduck.core.preferences.Preferences; -import ch.cyberduck.core.preferences.PreferencesFactory; - -import org.irods.jargon.core.packinstr.TransferOptions; - -public class DefaultTransferOptionsConfigurer { - - private final Preferences preferences = PreferencesFactory.get(); - - public TransferOptions configure(final TransferOptions options) { - options.setPutOption(TransferOptions.PutOptions.NORMAL); - options.setForceOption(TransferOptions.ForceOption.ASK_CALLBACK_LISTENER); - options.setMaxThreads(preferences.getInteger("queue.connections.limit.default")); - // Enable progress callbacks - options.setIntraFileStatusCallbacks(true); - options.setIntraFileStatusCallbacksNumberCallsInterval(1); - return options; - } -} diff --git a/irods/src/main/java/ch/cyberduck/core/irods/DefaultTransferStatusCallbackListener.java b/irods/src/main/java/ch/cyberduck/core/irods/DefaultTransferStatusCallbackListener.java deleted file mode 100644 index accf71879f9..00000000000 --- a/irods/src/main/java/ch/cyberduck/core/irods/DefaultTransferStatusCallbackListener.java +++ /dev/null @@ -1,90 +0,0 @@ -package ch.cyberduck.core.irods; - -/* - * Copyright (c) 2002-2015 David Kocher. All rights reserved. - * http://cyberduck.ch/ - * - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation; either version 2 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * Bug fixes, suggestions and comments should be sent to feedback@cyberduck.ch - */ - -import ch.cyberduck.core.BytecountStreamListener; -import ch.cyberduck.core.exception.ConnectionCanceledException; -import ch.cyberduck.core.io.StreamListener; -import ch.cyberduck.core.transfer.TransferStatus; - -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; -import org.irods.jargon.core.transfer.TransferControlBlock; -import org.irods.jargon.core.transfer.TransferStatusCallbackListener; - -public class DefaultTransferStatusCallbackListener implements TransferStatusCallbackListener { - private static final Logger log = LogManager.getLogger(DefaultTransferStatusCallbackListener.class); - - private final TransferStatus status; - private final BytecountStreamListener listener; - private final TransferControlBlock block; - - public DefaultTransferStatusCallbackListener(final TransferStatus status, final StreamListener listener, - final TransferControlBlock block) { - this.status = status; - this.listener = new BytecountStreamListener(listener); - this.block = block; - } - - @Override - public FileStatusCallbackResponse statusCallback(final org.irods.jargon.core.transfer.TransferStatus t) { - log.debug("Progress with {}", t); - final long bytes = t.getBytesTransfered() - listener.getSent(); - switch(t.getTransferType()) { - case GET: - listener.recv(bytes); - break; - case PUT: - listener.sent(bytes); - break; - } - try { - status.validate(); - if(!t.isIntraFileStatusReport()) { - if(t.getTotalFilesTransferredSoFar() == t.getTotalFilesToTransfer()) { - status.setComplete(); - } - } - } - catch(ConnectionCanceledException e) { - log.debug("Set canceled for block {}", block); - block.setCancelled(true); - return FileStatusCallbackResponse.SKIP; - } - return FileStatusCallbackResponse.CONTINUE; - } - - @Override - public void overallStatusCallback(final org.irods.jargon.core.transfer.TransferStatus t) { - // - } - - @Override - public CallbackResponse transferAsksWhetherToForceOperation(final String irodsAbsolutePath, final boolean isCollection) { - try { - status.validate(); - } - catch(ConnectionCanceledException e) { - return CallbackResponse.CANCEL; - } - if(status.isAppend()) { - return CallbackResponse.NO_THIS_FILE; - } - return CallbackResponse.YES_THIS_FILE; - } -} diff --git a/irods/src/main/java/ch/cyberduck/core/irods/IRODSAttributesFinderFeature.java b/irods/src/main/java/ch/cyberduck/core/irods/IRODSAttributesFinderFeature.java index 8abbc111bda..67dc93e433d 100644 --- a/irods/src/main/java/ch/cyberduck/core/irods/IRODSAttributesFinderFeature.java +++ b/irods/src/main/java/ch/cyberduck/core/irods/IRODSAttributesFinderFeature.java @@ -1,7 +1,7 @@ package ch.cyberduck.core.irods; /* - * Copyright (c) 2002-2016 iterate GmbH. All rights reserved. + * Copyright (c) 2002-2025 iterate GmbH. All rights reserved. * https://cyberduck.io/ * * This program is free software; you can redistribute it and/or modify @@ -15,6 +15,7 @@ * GNU General Public License for more details. */ +import ch.cyberduck.core.DefaultIOExceptionMappingService; import ch.cyberduck.core.ListProgressListener; import ch.cyberduck.core.Path; import ch.cyberduck.core.PathAttributes; @@ -22,16 +23,25 @@ import ch.cyberduck.core.exception.NotfoundException; import ch.cyberduck.core.features.AttributesAdapter; import ch.cyberduck.core.features.AttributesFinder; -import ch.cyberduck.core.io.Checksum; -import org.apache.commons.codec.binary.Base64; -import org.apache.commons.codec.binary.Hex; -import org.irods.jargon.core.exception.JargonException; -import org.irods.jargon.core.pub.IRODSFileSystemAO; -import org.irods.jargon.core.pub.domain.ObjStat; -import org.irods.jargon.core.pub.io.IRODSFile; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.irods.irods4j.high_level.catalog.IRODSQuery; +import org.irods.irods4j.high_level.connection.IRODSConnection; +import org.irods.irods4j.high_level.vfs.IRODSFilesystem; +import org.irods.irods4j.high_level.vfs.LogicalPath; +import org.irods.irods4j.high_level.vfs.ObjectStatus; +import org.irods.irods4j.low_level.api.IRODSException; -public class IRODSAttributesFinderFeature implements AttributesFinder, AttributesAdapter { +import java.io.IOException; +import java.util.List; + +public class IRODSAttributesFinderFeature implements AttributesFinder, AttributesAdapter> { + + private static final Logger log = LogManager.getLogger(IRODSAttributesFinderFeature.class); + + private static final String REPLICA_STATUS_GOOD = "1"; + private static final String REPLICA_STATUS_STALE = "0"; private final IRODSSession session; @@ -42,28 +52,79 @@ public IRODSAttributesFinderFeature(final IRODSSession session) { @Override public PathAttributes find(final Path file, final ListProgressListener listener) throws BackgroundException { try { - final IRODSFileSystemAO fs = session.getClient(); - final IRODSFile f = fs.getIRODSFileFactory().instanceIRODSFile(file.getAbsolute()); - if(!f.exists()) { - throw new NotfoundException(file.getAbsolute()); + log.debug("looking up path attributes."); + + final String logicalPath = file.getAbsolute(); + final IRODSConnection conn = session.getClient(); + + ObjectStatus status = IRODSFilesystem.status(conn.getRcComm(), logicalPath); + + if(IRODSFilesystem.isDataObject(status)) { + log.debug("data object exists in iRODS. fetching data using GenQuery2."); + String query = String.format( + "select DATA_CREATE_TIME, DATA_MODIFY_TIME, DATA_SIZE, DATA_CHECKSUM, DATA_REPL_STATUS where COLL_NAME = '%s' and DATA_NAME = '%s' order by DATA_REPL_STATUS desc, DATA_MODIFY_TIME desc", + LogicalPath.parentPath(logicalPath), + LogicalPath.objectName(logicalPath)); + log.debug("query = [{}]", query); + List> rows = IRODSQuery.executeGenQuery2(conn.getRcComm(), query); + + PathAttributes attrs = new PathAttributes(); + + if(!rows.isEmpty()) { + List row = rows.get(0); + if(REPLICA_STATUS_STALE.equals(row.get(4)) || REPLICA_STATUS_GOOD.equals(row.get(4))) { + setAttributes(attrs, row); + } + } + + return attrs; } - final ObjStat stats = fs.getObjStat(f.getAbsolutePath()); - return this.toAttributes(stats); + + if(IRODSFilesystem.isCollection(status)) { + log.debug("collection exists in iRODS. fetching data using GenQuery2."); + String query = String.format("select COLL_CREATE_TIME, COLL_MODIFY_TIME where COLL_NAME = '%s'", logicalPath); + log.debug("query = [{}]", query); + List> rows = IRODSQuery.executeGenQuery2(conn.getRcComm(), query); + + PathAttributes attrs = new PathAttributes(); + + if(!rows.isEmpty()) { + // Collections do not have the same properties as data objects + // so fill in the gaps to satisfy requirements of setAttributes. + List row = rows.get(0); + row.add("0"); // Data size + row.add(""); // Checksum + row.add(""); // Replica status + setAttributes(attrs, row); + } + + return attrs; + } + + throw new NotfoundException(logicalPath); } - catch(JargonException e) { + catch(IRODSException e) { throw new IRODSExceptionMappingService().map("Failure to read attributes of {0}", e, file); } + catch(IOException e) { + throw new DefaultIOExceptionMappingService().map("Failure to read attributes of {0}", e, file); + } } @Override - public PathAttributes toAttributes(final ObjStat stats) { - final PathAttributes attributes = new PathAttributes(); - attributes.setModificationDate(stats.getModifiedAt().getTime()); - attributes.setCreationDate(stats.getCreatedAt().getTime()); - attributes.setSize(stats.getObjSize()); - attributes.setChecksum(Checksum.parse(Hex.encodeHexString(Base64.decodeBase64(stats.getChecksum())))); - attributes.setOwner(stats.getOwnerName()); - attributes.setGroup(stats.getOwnerZone()); - return attributes; + public PathAttributes toAttributes(final List row) { + PathAttributes attrs = new PathAttributes(); + setAttributes(attrs, row); + return attrs; + } + + private static void setAttributes(final PathAttributes attrs, final List row) { + log.debug("path attribute info: created at [{}], modified at [{}], data size = [{}], checksum = [{}]", + row.get(0), row.get(1), row.get(2), row.get(3)); + attrs.setCreationDate(Long.parseLong(row.get(0)) * 1000); // seconds to ms + attrs.setModificationDate(Long.parseLong(row.get(1)) * 1000); + attrs.setSize(Long.parseLong(row.get(2))); + attrs.setChecksum(IRODSChecksumUtils.toChecksum(row.get(3))); } + } diff --git a/irods/src/main/java/ch/cyberduck/core/irods/IRODSChecksumUtils.java b/irods/src/main/java/ch/cyberduck/core/irods/IRODSChecksumUtils.java new file mode 100644 index 00000000000..ddd41ff66fd --- /dev/null +++ b/irods/src/main/java/ch/cyberduck/core/irods/IRODSChecksumUtils.java @@ -0,0 +1,52 @@ +package ch.cyberduck.core.irods; + +/* + * Copyright (c) 2002-2025 iterate GmbH. All rights reserved. + * https://cyberduck.io/ + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + */ + +import ch.cyberduck.core.io.Checksum; + +import org.apache.commons.codec.binary.Base64; +import org.apache.commons.codec.binary.Hex; +import org.apache.commons.lang3.StringUtils; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +public final class IRODSChecksumUtils { + + private static final Logger log = LogManager.getLogger(IRODSChecksumUtils.class); + + public static Checksum toChecksum(String irodsChecksum) { + if(StringUtils.isBlank(irodsChecksum)) { + return Checksum.NONE; + } + + int colon = irodsChecksum.indexOf(':'); + if(-1 == colon) { + log.debug("no hash algorithm prefix found in iRODS checksum. ignoring checksum."); + return Checksum.NONE; + } + + if(colon + 1 >= irodsChecksum.length()) { + log.debug("iRODS checksum may be corrupted. ignoring checksum."); + return Checksum.NONE; + } + + log.debug("checksum from iRODS server is [{}].", irodsChecksum); + String checksum = irodsChecksum.substring(colon + 1); + checksum = Hex.encodeHexString(Base64.decodeBase64(checksum)); + log.debug("base64-decoded, hex-encoded checksum is [{}].", checksum); + return Checksum.parse(checksum); + } +} diff --git a/irods/src/main/java/ch/cyberduck/core/irods/IRODSChunkWorker.java b/irods/src/main/java/ch/cyberduck/core/irods/IRODSChunkWorker.java new file mode 100644 index 00000000000..0e33be839bf --- /dev/null +++ b/irods/src/main/java/ch/cyberduck/core/irods/IRODSChunkWorker.java @@ -0,0 +1,101 @@ +package ch.cyberduck.core.irods; + +/* + * Copyright (c) 2002-2025 iterate GmbH. All rights reserved. + * https://cyberduck.io/ + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + */ + +import ch.cyberduck.core.exception.ConnectionCanceledException; +import ch.cyberduck.core.io.StreamListener; +import ch.cyberduck.core.transfer.TransferStatus; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.irods.irods4j.low_level.api.IRODSException; + +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.util.concurrent.Callable; + +public class IRODSChunkWorker implements Callable { + + private static final Logger log = LogManager.getLogger(IRODSChunkWorker.class); + + private final TransferStatus status; + private final StreamListener streamListener; + private final InputStream in; + private final OutputStream out; + private final long offset; + private final long chunkSize; + private final byte[] buffer; + + public IRODSChunkWorker(TransferStatus status, StreamListener streamListener, InputStream in, OutputStream out, long offset, long chunkSize, int bufferSize) { + log.info("constructing iRODS chunk worker."); + log.info("offset = [{}]", offset); + log.info("chunk size = [{}]", chunkSize); + log.info("buffer size = [{}]", bufferSize); + this.status = status; + this.streamListener = streamListener; + this.in = in; + this.out = out; + this.offset = offset; + this.chunkSize = chunkSize; + this.buffer = new byte[bufferSize]; + log.info("iRODS chunk worker constructed."); + } + + @Override + public Boolean call() { + try { + IRODSStreamUtils.seek(in, offset); + IRODSStreamUtils.seek(out, offset); + + long remaining = chunkSize; + while(remaining > 0) { + try { + status.validate(); + } + catch(ConnectionCanceledException e) { + log.info("transfer cancelled."); + return false; + } + + int count = (int) Math.min(buffer.length, remaining); + + int bytesRead = in.read(buffer, 0, count); + log.info("read [{}] of [{}] requested bytes from input stream.", bytesRead, count); + if(-1 == bytesRead) { + break; + } + + streamListener.recv(bytesRead); + out.write(buffer, 0, bytesRead); + log.info("wrote [{}] bytes to output stream.", bytesRead); + streamListener.sent(bytesRead); + remaining -= bytesRead; + } + + log.info("total bytes remaining = [{}]", remaining); + log.info("done. wrote [{}] of [{}] bytes to the replica.", chunkSize - remaining, chunkSize); + + return true; + } + catch(IOException | IRODSException e) { + log.error(e.getMessage()); + } + + return false; + } + +} diff --git a/irods/src/main/java/ch/cyberduck/core/irods/IRODSConnectionUtils.java b/irods/src/main/java/ch/cyberduck/core/irods/IRODSConnectionUtils.java new file mode 100644 index 00000000000..918589ef003 --- /dev/null +++ b/irods/src/main/java/ch/cyberduck/core/irods/IRODSConnectionUtils.java @@ -0,0 +1,138 @@ +package ch.cyberduck.core.irods; + +/* + * Copyright (c) 2002-2025 iterate GmbH. All rights reserved. + * https://cyberduck.io/ + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + */ + +import ch.cyberduck.core.preferences.HostPreferencesFactory; +import ch.cyberduck.core.preferences.PreferencesReader; + +import org.apache.commons.lang3.StringUtils; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.irods.irods4j.authentication.AuthPlugin; +import org.irods.irods4j.authentication.NativeAuthPlugin; +import org.irods.irods4j.authentication.PamPasswordAuthPlugin; +import org.irods.irods4j.high_level.connection.IRODSConnection; +import org.irods.irods4j.high_level.connection.IRODSConnectionPool; +import org.irods.irods4j.high_level.connection.QualifiedUsername; +import org.irods.irods4j.low_level.api.IRODSApi; +import org.irods.irods4j.low_level.api.IRODSException; + +import java.io.IOException; + +final class IRODSConnectionUtils { + + private static final Logger log = LogManager.getLogger(IRODSConnectionUtils.class); + + private enum AuthScheme { + NATIVE, + PAM_PASSWORD; + + public static AuthScheme fromString(String s) { + if(null == s) { + throw new IllegalArgumentException("Cannot convert null to AuthScheme"); + } + + if("native".equalsIgnoreCase(s) || "standard".equalsIgnoreCase(s)) { + return NATIVE; + } + + if("pam_password".equalsIgnoreCase(s)) { + return PAM_PASSWORD; + } + + throw new IllegalArgumentException(String.format("Cannot convert [%s] to AuthScheme", s)); + } + } + + public static IRODSApi.ConnectionOptions initConnectionOptions(IRODSSession session) { + log.debug("configuring iRODS connection."); + final PreferencesReader preferences = HostPreferencesFactory.get(session.getHost()); + final IRODSApi.ConnectionOptions options = new IRODSApi.ConnectionOptions(); + + options.clientServerNegotiation = preferences.getProperty(IRODSProtocol.CLIENT_SERVER_NEGOTIATION); + options.sslProtocol = preferences.getProperty(IRODSProtocol.TLS_PROTOCOL); + options.sslTruststore = preferences.getProperty(IRODSProtocol.TLS_TRUSTSTORE); + options.sslTruststorePassword = preferences.getProperty(IRODSProtocol.TLS_TRUSTSTORE_PASSWORD); + log.debug("client server negotiation = [{}], ssl protocol = [{}], ssl truststore = [{}]", + options.clientServerNegotiation, options.sslProtocol, options.sslTruststore); + + options.encryptionAlgorithm = preferences.getProperty(IRODSProtocol.ENCRYPTION_ALGORITHM); + options.encryptionKeySize = preferences.getInteger(IRODSProtocol.ENCRYPTION_KEY_SIZE); + options.encryptionSaltSize = preferences.getInteger(IRODSProtocol.ENCRYPTION_SALT_SIZE); + options.encryptionNumHashRounds = preferences.getInteger(IRODSProtocol.ENCRYPTION_HASH_ROUNDS); + log.debug("encryption algorithm = [{}], encryption key size = [{}], encryption salt size = [{}], encryption hash rounds = [{}]", + options.encryptionAlgorithm, options.encryptionKeySize, options.encryptionSaltSize, options.encryptionNumHashRounds); + + return options; + } + + public static AuthPlugin newAuthPlugin(IRODSSession session) { + AuthPlugin plugin = null; + + final String authSchemeStr = StringUtils.defaultIfBlank(session.getHost().getProtocol().getAuthorization(), "native"); + final AuthScheme authScheme = AuthScheme.fromString(authSchemeStr); + switch(authScheme) { + case NATIVE: + plugin = new NativeAuthPlugin(); + break; + + case PAM_PASSWORD: + plugin = new PamPasswordAuthPlugin(true); + break; + + default: + // Should never get here. + throw new IllegalArgumentException("Cannot resolve authentication scheme to plugin implementation"); + } + + return plugin; + } + + public static IRODSConnection newAuthenticatedConnection(IRODSSession session) throws Exception { + String host = session.getHost().getHostname(); + int port = session.getHost().getPort(); + String zone = session.getRegion(); + String username = session.getHost().getCredentials().getUsername(); + String password = session.getHost().getCredentials().getPassword(); + IRODSConnection conn = new IRODSConnection(initConnectionOptions(session)); + conn.connect(host, port, new QualifiedUsername(username, zone)); + conn.authenticate(newAuthPlugin(session), password); + return conn; + } + + public static void startIRODSConnectionPool(IRODSSession session, IRODSConnectionPool connPool) throws IRODSException, IOException { + String host = session.getHost().getHostname(); + int port = session.getHost().getPort(); + String zone = session.getRegion(); + String username = session.getHost().getCredentials().getUsername(); + String password = session.getHost().getCredentials().getPassword(); + + connPool.start( + host, + port, + new QualifiedUsername(username, zone), + conn -> { + try { + IRODSApi.rcAuthenticateClient(conn, newAuthPlugin(session), password); + return true; + } + catch(Exception e) { + log.error(e.getMessage()); + return false; + } + }); + } +} diff --git a/irods/src/main/java/ch/cyberduck/core/irods/IRODSCopyFeature.java b/irods/src/main/java/ch/cyberduck/core/irods/IRODSCopyFeature.java index 0687bdf24d7..30674f5e9ae 100644 --- a/irods/src/main/java/ch/cyberduck/core/irods/IRODSCopyFeature.java +++ b/irods/src/main/java/ch/cyberduck/core/irods/IRODSCopyFeature.java @@ -1,8 +1,8 @@ package ch.cyberduck.core.irods; /* - * Copyright (c) 2002-2015 David Kocher. All rights reserved. - * http://cyberduck.ch/ + * Copyright (c) 2002-2025 iterate GmbH. All rights reserved. + * https://cyberduck.io/ * * This program is free software; you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by @@ -13,25 +13,22 @@ * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * GNU General Public License for more details. - * - * Bug fixes, suggestions and comments should be sent to feedback@cyberduck.ch */ import ch.cyberduck.core.ConnectionCallback; +import ch.cyberduck.core.DefaultIOExceptionMappingService; import ch.cyberduck.core.Path; import ch.cyberduck.core.exception.BackgroundException; import ch.cyberduck.core.features.Copy; import ch.cyberduck.core.io.StreamListener; -import ch.cyberduck.core.preferences.HostPreferencesFactory; -import org.apache.commons.lang3.StringUtils; -import org.irods.jargon.core.exception.JargonException; -import org.irods.jargon.core.pub.DataTransferOperations; -import org.irods.jargon.core.pub.IRODSFileSystemAO; -import org.irods.jargon.core.transfer.DefaultTransferControlBlock; -import org.irods.jargon.core.transfer.TransferStatus; -import org.irods.jargon.core.transfer.TransferStatusCallbackListener; +import ch.cyberduck.core.transfer.TransferStatus; + +import org.irods.irods4j.high_level.connection.IRODSConnection; +import org.irods.irods4j.high_level.vfs.IRODSFilesystem; +import org.irods.irods4j.low_level.api.IRODSException; +import java.io.IOException; import java.util.EnumSet; public class IRODSCopyFeature implements Copy { @@ -43,37 +40,28 @@ public IRODSCopyFeature(final IRODSSession session) { } @Override - public Path copy(final Path source, final Path target, final ch.cyberduck.core.transfer.TransferStatus status, final ConnectionCallback callback, final StreamListener listener) throws BackgroundException { + public Path copy(final Path source, final Path target, final TransferStatus status, + final ConnectionCallback callback, final StreamListener listener) throws BackgroundException { try { - final IRODSFileSystemAO fs = session.getClient(); - final DataTransferOperations transfer = fs.getIRODSAccessObjectFactory() - .getDataTransferOperations(fs.getIRODSAccount()); - transfer.copy(fs.getIRODSFileFactory().instanceIRODSFile(source.getAbsolute()), - fs.getIRODSFileFactory().instanceIRODSFile(target.getAbsolute()), new TransferStatusCallbackListener() { - @Override - public FileStatusCallbackResponse statusCallback(final TransferStatus transferStatus) { - return FileStatusCallbackResponse.CONTINUE; - } + final IRODSConnection conn = session.getClient(); + final String from = source.getAbsolute(); + final String to = target.getAbsolute(); - @Override - public void overallStatusCallback(final TransferStatus transferStatus) { - switch(transferStatus.getTransferState()) { - case OVERALL_COMPLETION: - listener.sent(status.getLength()); - } - } + int options = IRODSFilesystem.CopyOptions.RECURSIVE; + if(status.isExists()) { + options |= IRODSFilesystem.CopyOptions.OVERWRITE_EXISTING; + } + + IRODSFilesystem.copy(conn.getRcComm(), from, to, options); - @Override - public CallbackResponse transferAsksWhetherToForceOperation(final String irodsAbsolutePath, final boolean isCollection) { - return CallbackResponse.YES_THIS_FILE; - } - }, DefaultTransferControlBlock.instance(StringUtils.EMPTY, - HostPreferencesFactory.get(session.getHost()).getInteger("connection.retry"))); return target; } - catch(JargonException e) { + catch(IRODSException e) { throw new IRODSExceptionMappingService().map("Cannot copy {0}", e, source); } + catch(IOException e) { + throw new DefaultIOExceptionMappingService().map("Cannot copy {0}", e, source); + } } @Override diff --git a/irods/src/main/java/ch/cyberduck/core/irods/IRODSDeleteFeature.java b/irods/src/main/java/ch/cyberduck/core/irods/IRODSDeleteFeature.java index 47261db3efa..3bf9d376358 100644 --- a/irods/src/main/java/ch/cyberduck/core/irods/IRODSDeleteFeature.java +++ b/irods/src/main/java/ch/cyberduck/core/irods/IRODSDeleteFeature.java @@ -1,8 +1,8 @@ package ch.cyberduck.core.irods; /* - * Copyright (c) 2002-2015 David Kocher. All rights reserved. - * http://cyberduck.ch/ + * Copyright (c) 2002-2025 iterate GmbH. All rights reserved. + * https://cyberduck.io/ * * This program is free software; you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by @@ -13,20 +13,26 @@ * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * GNU General Public License for more details. - * - * Bug fixes, suggestions and comments should be sent to feedback@cyberduck.ch */ +import ch.cyberduck.core.DefaultIOExceptionMappingService; import ch.cyberduck.core.PasswordCallback; import ch.cyberduck.core.Path; import ch.cyberduck.core.exception.BackgroundException; import ch.cyberduck.core.exception.NotfoundException; import ch.cyberduck.core.features.Delete; +import ch.cyberduck.core.preferences.HostPreferencesFactory; +import ch.cyberduck.core.preferences.PreferencesReader; import ch.cyberduck.core.transfer.TransferStatus; -import org.irods.jargon.core.exception.JargonException; -import org.irods.jargon.core.pub.io.IRODSFile; +import org.irods.irods4j.high_level.connection.IRODSConnection; +import org.irods.irods4j.high_level.vfs.IRODSFilesystem; +import org.irods.irods4j.high_level.vfs.IRODSFilesystem.RemoveOptions; +import org.irods.irods4j.high_level.vfs.ObjectStatus; +import org.irods.irods4j.high_level.vfs.ObjectStatus.ObjectType; +import org.irods.irods4j.low_level.api.IRODSException; +import java.io.IOException; import java.util.ArrayList; import java.util.EnumSet; import java.util.List; @@ -35,9 +41,18 @@ public class IRODSDeleteFeature implements Delete { private final IRODSSession session; + private final RemoveOptions removeOptions; public IRODSDeleteFeature(IRODSSession session) { this.session = session; + + PreferencesReader prefs = HostPreferencesFactory.get(session.getHost()); + if(prefs.getBoolean(IRODSProtocol.DELETE_OBJECTS_PERMANTENTLY)) { + removeOptions = RemoveOptions.NO_TRASH; + } + else { + removeOptions = RemoveOptions.NONE; + } } @Override @@ -51,26 +66,36 @@ public void delete(final Map files, final PasswordCallback break; } } + if(skip) { continue; } + deleted.add(file); callback.delete(file); + try { - final IRODSFile f = session.getClient().getIRODSFileFactory().instanceIRODSFile(file.getAbsolute()); - if(!f.exists()) { - throw new NotfoundException(String.format("%s doesn't exist", file.getAbsolute())); + final IRODSConnection conn = session.getClient(); + final String logicalPath = file.getAbsolute(); + final ObjectStatus status = IRODSFilesystem.status(conn.getRcComm(), logicalPath); + + if(!IRODSFilesystem.exists(status)) { + throw new NotfoundException(String.format("%s doesn't exist", logicalPath)); } - if(f.isFile()) { - session.getClient().fileDeleteForce(f); + + if(status.getType() == ObjectType.DATA_OBJECT) { + IRODSFilesystem.remove(conn.getRcComm(), logicalPath, removeOptions); } - else if(f.isDirectory()) { - session.getClient().directoryDeleteForce(f); + else if(status.getType() == ObjectType.COLLECTION) { + IRODSFilesystem.removeAll(conn.getRcComm(), logicalPath, removeOptions); } } - catch(JargonException e) { + catch(IRODSException e) { throw new IRODSExceptionMappingService().map("Cannot delete {0}", e, file); } + catch(IOException e) { + throw new DefaultIOExceptionMappingService().map("Cannot delete {0}", e, file); + } } } diff --git a/irods/src/main/java/ch/cyberduck/core/irods/IRODSDirectoryFeature.java b/irods/src/main/java/ch/cyberduck/core/irods/IRODSDirectoryFeature.java index 7f4e6ee90da..746990a36b9 100644 --- a/irods/src/main/java/ch/cyberduck/core/irods/IRODSDirectoryFeature.java +++ b/irods/src/main/java/ch/cyberduck/core/irods/IRODSDirectoryFeature.java @@ -1,8 +1,8 @@ package ch.cyberduck.core.irods; /* - * Copyright (c) 2002-2015 David Kocher. All rights reserved. - * http://cyberduck.ch/ + * Copyright (c) 2002-2025 iterate GmbH. All rights reserved. + * https://cyberduck.io/ * * This program is free software; you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by @@ -13,19 +13,20 @@ * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * GNU General Public License for more details. - * - * Bug fixes, suggestions and comments should be sent to feedback@cyberduck.ch */ +import ch.cyberduck.core.DefaultIOExceptionMappingService; import ch.cyberduck.core.Path; import ch.cyberduck.core.exception.BackgroundException; import ch.cyberduck.core.features.Directory; import ch.cyberduck.core.features.Write; import ch.cyberduck.core.transfer.TransferStatus; -import org.irods.jargon.core.exception.JargonException; -import org.irods.jargon.core.pub.IRODSFileSystemAO; -import org.irods.jargon.core.pub.io.IRODSFile; +import org.irods.irods4j.high_level.connection.IRODSConnection; +import org.irods.irods4j.high_level.vfs.IRODSFilesystem; +import org.irods.irods4j.high_level.vfs.IRODSFilesystemException; + +import java.io.IOException; public class IRODSDirectoryFeature implements Directory { @@ -38,13 +39,15 @@ public IRODSDirectoryFeature(final IRODSSession session) { @Override public Path mkdir(final Write writer, final Path folder, final TransferStatus status) throws BackgroundException { try { - final IRODSFileSystemAO fs = session.getClient(); - final IRODSFile f = fs.getIRODSFileFactory().instanceIRODSFile(folder.getAbsolute()); - fs.mkdir(f, false); + final IRODSConnection conn = session.getClient(); + IRODSFilesystem.createCollection(conn.getRcComm(), folder.getAbsolute()); return folder; } - catch(JargonException e) { + catch(IRODSFilesystemException e) { throw new IRODSExceptionMappingService().map("Cannot create folder {0}", e, folder); } + catch(IOException e) { + throw new DefaultIOExceptionMappingService().map("Cannot create folder {0}", e, folder); + } } } diff --git a/irods/src/main/java/ch/cyberduck/core/irods/IRODSDownloadFeature.java b/irods/src/main/java/ch/cyberduck/core/irods/IRODSDownloadFeature.java deleted file mode 100644 index bcf7b24e554..00000000000 --- a/irods/src/main/java/ch/cyberduck/core/irods/IRODSDownloadFeature.java +++ /dev/null @@ -1,90 +0,0 @@ -package ch.cyberduck.core.irods; - -/* - * Copyright (c) 2002-2015 David Kocher. All rights reserved. - * http://cyberduck.ch/ - * - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation; either version 2 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * Bug fixes, suggestions and comments should be sent to feedback@cyberduck.ch - */ - -import ch.cyberduck.core.ConnectionCallback; -import ch.cyberduck.core.Host; -import ch.cyberduck.core.Local; -import ch.cyberduck.core.Path; -import ch.cyberduck.core.exception.BackgroundException; -import ch.cyberduck.core.exception.NotfoundException; -import ch.cyberduck.core.features.Download; -import ch.cyberduck.core.features.Read; -import ch.cyberduck.core.io.BandwidthThrottle; -import ch.cyberduck.core.io.StreamListener; -import ch.cyberduck.core.preferences.HostPreferencesFactory; -import ch.cyberduck.core.preferences.PreferencesFactory; -import ch.cyberduck.core.transfer.TransferStatus; - -import org.apache.commons.lang3.StringUtils; -import org.irods.jargon.core.exception.JargonException; -import org.irods.jargon.core.packinstr.TransferOptions; -import org.irods.jargon.core.pub.DataTransferOperations; -import org.irods.jargon.core.pub.IRODSFileSystemAO; -import org.irods.jargon.core.pub.io.IRODSFile; -import org.irods.jargon.core.transfer.DefaultTransferControlBlock; -import org.irods.jargon.core.transfer.TransferControlBlock; - -import java.io.File; - -public class IRODSDownloadFeature implements Download { - - private final IRODSSession session; - - public IRODSDownloadFeature(final IRODSSession session) { - this.session = session; - } - - @Override - public void download(final Read read, final Path file, final Local local, final BandwidthThrottle throttle, - final StreamListener listener, final TransferStatus status, - final ConnectionCallback callback) throws BackgroundException { - try { - final IRODSFileSystemAO fs = session.getClient(); - final IRODSFile f = fs.getIRODSFileFactory().instanceIRODSFile(file.getAbsolute()); - if(f.exists()) { - final TransferControlBlock block = DefaultTransferControlBlock.instance(StringUtils.EMPTY, - HostPreferencesFactory.get(session.getHost()).getInteger("connection.retry")); - final TransferOptions options = new DefaultTransferOptionsConfigurer().configure(new TransferOptions()); - if(Host.TransferType.unknown.equals(session.getHost().getTransferType())) { - options.setUseParallelTransfer(Host.TransferType.valueOf(PreferencesFactory.get().getProperty("queue.transfer.type")).equals(Host.TransferType.concurrent)); - } - else { - options.setUseParallelTransfer(session.getHost().getTransferType().equals(Host.TransferType.concurrent)); - } - block.setTransferOptions(options); - final DataTransferOperations transfer = fs.getIRODSAccessObjectFactory() - .getDataTransferOperations(fs.getIRODSAccount()); - transfer.getOperation(f, new File(local.getAbsolute()), - new DefaultTransferStatusCallbackListener(status, listener, block), - block); - } - else { - throw new NotfoundException(file.getAbsolute()); - } - } - catch(JargonException e) { - throw new IRODSExceptionMappingService().map("Download {0} failed", e, file); - } - } - - @Override - public boolean offset(final Path file) { - return false; - } -} diff --git a/irods/src/main/java/ch/cyberduck/core/irods/IRODSExceptionMappingService.java b/irods/src/main/java/ch/cyberduck/core/irods/IRODSExceptionMappingService.java index 244566915b0..da5dfa0840a 100644 --- a/irods/src/main/java/ch/cyberduck/core/irods/IRODSExceptionMappingService.java +++ b/irods/src/main/java/ch/cyberduck/core/irods/IRODSExceptionMappingService.java @@ -1,8 +1,8 @@ package ch.cyberduck.core.irods; /* - * Copyright (c) 2002-2015 David Kocher. All rights reserved. - * http://cyberduck.ch/ + * Copyright (c) 2002-2025 iterate GmbH. All rights reserved. + * https://cyberduck.io/ * * This program is free software; you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by @@ -13,52 +13,70 @@ * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * GNU General Public License for more details. - * - * Bug fixes, suggestions and comments should be sent to feedback@cyberduck.ch */ import ch.cyberduck.core.AbstractExceptionMappingService; import ch.cyberduck.core.exception.AccessDeniedException; import ch.cyberduck.core.exception.BackgroundException; + +import ch.cyberduck.core.exception.ConnectionRefusedException; +import ch.cyberduck.core.exception.LockedException; import ch.cyberduck.core.exception.LoginFailureException; + import ch.cyberduck.core.exception.NotfoundException; +import ch.cyberduck.core.exception.QuotaException; +import ch.cyberduck.core.exception.SSLNegotiateException; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; -import org.irods.jargon.core.exception.AuthenticationException; -import org.irods.jargon.core.exception.CatNoAccessException; -import org.irods.jargon.core.exception.DataNotFoundException; -import org.irods.jargon.core.exception.FileNotFoundException; -import org.irods.jargon.core.exception.InvalidGroupException; -import org.irods.jargon.core.exception.InvalidUserException; -import org.irods.jargon.core.exception.JargonException; - -public class IRODSExceptionMappingService extends AbstractExceptionMappingService { +import org.irods.irods4j.low_level.api.IRODSErrorCodes; +import org.irods.irods4j.low_level.api.IRODSException; + +public class IRODSExceptionMappingService extends AbstractExceptionMappingService { + private static final Logger log = LogManager.getLogger(IRODSExceptionMappingService.class); @Override - public BackgroundException map(final JargonException e) { + public BackgroundException map(final IRODSException e) { log.warn("Map failure {}", e.toString()); final StringBuilder buffer = new StringBuilder(); this.append(buffer, e.getMessage()); - if(e instanceof CatNoAccessException) { - return new AccessDeniedException(buffer.toString(), e); - } - if(e instanceof FileNotFoundException) { - return new NotfoundException(buffer.toString(), e); - } - if(e instanceof DataNotFoundException) { - return new NotfoundException(buffer.toString(), e); - } - if(e instanceof AuthenticationException) { - return new LoginFailureException(buffer.toString(), e); - } - if(e instanceof InvalidUserException) { - return new LoginFailureException(buffer.toString(), e); - } - if(e instanceof InvalidGroupException) { - return new LoginFailureException(buffer.toString(), e); + + switch(e.getErrorCode()) { + case IRODSErrorCodes.AUTHENTICATION_ERROR: + case IRODSErrorCodes.CAT_INVALID_AUTHENTICATION: + case IRODSErrorCodes.CAT_PASSWORD_EXPIRED: + case IRODSErrorCodes.CAT_INVALID_USER: + return new LoginFailureException(buffer.toString(), e); + + case IRODSErrorCodes.CAT_NO_ACCESS_PERMISSION: + case IRODSErrorCodes.CAT_INSUFFICIENT_PRIVILEGE_LEVEL: + case IRODSErrorCodes.SYS_NOT_ALLOWED: + return new AccessDeniedException(buffer.toString(), e); + + case IRODSErrorCodes.INTERMEDIATE_REPLICA_ACCESS: + case IRODSErrorCodes.SYS_REPLICA_INACCESSIBLE: + return new LockedException(buffer.toString(), e); + + case IRODSErrorCodes.SSL_CERT_ERROR: + case IRODSErrorCodes.SSL_HANDSHAKE_ERROR: + case IRODSErrorCodes.SSL_INIT_ERROR: + case IRODSErrorCodes.SSL_SHUTDOWN_ERROR: + case IRODSErrorCodes.SSL_NOT_BUILT_INTO_CLIENT: + case IRODSErrorCodes.SSL_NOT_BUILT_INTO_SERVER: + return new SSLNegotiateException(buffer.toString(), e); + + case IRODSErrorCodes.SYS_RESC_QUOTA_EXCEEDED: + return new QuotaException(buffer.toString(), e); + + case IRODSErrorCodes.CAT_NO_ROWS_FOUND: + case IRODSErrorCodes.CAT_NOT_A_DATAOBJ_AND_NOT_A_COLLECTION: + return new NotfoundException(buffer.toString(), e); + + case IRODSErrorCodes.CONNECTION_REFUSED: + return new ConnectionRefusedException(buffer.toString(), e); } + return this.wrap(e, buffer); } } diff --git a/irods/src/main/java/ch/cyberduck/core/irods/IRODSFindFeature.java b/irods/src/main/java/ch/cyberduck/core/irods/IRODSFindFeature.java index ee3154370e9..78ca3b86ea8 100644 --- a/irods/src/main/java/ch/cyberduck/core/irods/IRODSFindFeature.java +++ b/irods/src/main/java/ch/cyberduck/core/irods/IRODSFindFeature.java @@ -1,8 +1,8 @@ package ch.cyberduck.core.irods; /* - * Copyright (c) 2002-2015 David Kocher. All rights reserved. - * http://cyberduck.ch/ + * Copyright (c) 2002-2025 iterate GmbH. All rights reserved. + * https://cyberduck.io/ * * This program is free software; you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by @@ -13,18 +13,19 @@ * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * GNU General Public License for more details. - * - * Bug fixes, suggestions and comments should be sent to feedback@cyberduck.ch */ +import ch.cyberduck.core.DefaultIOExceptionMappingService; import ch.cyberduck.core.ListProgressListener; import ch.cyberduck.core.Path; import ch.cyberduck.core.exception.BackgroundException; import ch.cyberduck.core.features.Find; -import org.irods.jargon.core.exception.JargonException; -import org.irods.jargon.core.pub.IRODSFileSystemAO; -import org.irods.jargon.core.pub.io.IRODSFile; +import org.irods.irods4j.high_level.connection.IRODSConnection; +import org.irods.irods4j.high_level.vfs.IRODSFilesystem; +import org.irods.irods4j.low_level.api.IRODSException; + +import java.io.IOException; public class IRODSFindFeature implements Find { @@ -39,13 +40,16 @@ public boolean find(final Path file, final ListProgressListener listener) throws if(file.isRoot()) { return true; } + try { - final IRODSFileSystemAO fs = session.getClient(); - final IRODSFile f = fs.getIRODSFileFactory().instanceIRODSFile(file.getAbsolute()); - return fs.isFileExists(f); + final IRODSConnection conn = session.getClient(); + return IRODSFilesystem.exists(conn.getRcComm(), file.getAbsolute()); + } + catch(IRODSException e) { + throw new IRODSExceptionMappingService().map("Failure to find {0}", e, file); } - catch(JargonException e) { - throw new IRODSExceptionMappingService().map("Failure to read attributes of {0}", e, file); + catch(IOException e) { + throw new DefaultIOExceptionMappingService().map("Failure to find {0}", e, file); } } } diff --git a/irods/src/main/java/ch/cyberduck/core/irods/IRODSHomeFinderService.java b/irods/src/main/java/ch/cyberduck/core/irods/IRODSHomeFinderService.java index 6720017524d..8833694c733 100644 --- a/irods/src/main/java/ch/cyberduck/core/irods/IRODSHomeFinderService.java +++ b/irods/src/main/java/ch/cyberduck/core/irods/IRODSHomeFinderService.java @@ -1,7 +1,7 @@ package ch.cyberduck.core.irods; /* - * Copyright (c) 2002-2016 iterate GmbH. All rights reserved. + * Copyright (c) 2002-2025 iterate GmbH. All rights reserved. * https://cyberduck.io/ * * This program is free software; you can redistribute it and/or modify @@ -36,16 +36,18 @@ public IRODSHomeFinderService(final IRODSSession session) { public Path find() throws BackgroundException { final String user; final Credentials credentials = session.getHost().getCredentials(); + if(StringUtils.contains(credentials.getUsername(), ':')) { user = StringUtils.splitPreserveAllTokens(credentials.getUsername(), ':')[1]; } else { user = credentials.getUsername(); } + return new Path(new StringBuilder() - .append(Path.DELIMITER).append(session.getRegion()) - .append(Path.DELIMITER).append("home") - .append(Path.DELIMITER).append(user) - .toString(), EnumSet.of(Path.Type.directory, Path.Type.volume)); + .append(Path.DELIMITER).append(session.getRegion()) + .append(Path.DELIMITER).append("home") + .append(Path.DELIMITER).append(user) + .toString(), EnumSet.of(Path.Type.directory, Path.Type.volume)); } } diff --git a/irods/src/main/java/ch/cyberduck/core/irods/IRODSIntegerUtils.java b/irods/src/main/java/ch/cyberduck/core/irods/IRODSIntegerUtils.java new file mode 100644 index 00000000000..99b2c40ef0f --- /dev/null +++ b/irods/src/main/java/ch/cyberduck/core/irods/IRODSIntegerUtils.java @@ -0,0 +1,32 @@ +package ch.cyberduck.core.irods; + +/* + * Copyright (c) 2002-2025 iterate GmbH. All rights reserved. + * https://cyberduck.io/ + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + */ + +public final class IRODSIntegerUtils { + + static > T clamp(T value, T low, T high) { + if(value.compareTo(low) < 0) { + return low; + } + + if(value.compareTo(high) > 0) { + return high; + } + + return value; + } + +} diff --git a/irods/src/main/java/ch/cyberduck/core/irods/IRODSListService.java b/irods/src/main/java/ch/cyberduck/core/irods/IRODSListService.java index 9df21070b9e..228a8f94bef 100644 --- a/irods/src/main/java/ch/cyberduck/core/irods/IRODSListService.java +++ b/irods/src/main/java/ch/cyberduck/core/irods/IRODSListService.java @@ -1,8 +1,8 @@ package ch.cyberduck.core.irods; /* - * Copyright (c) 2002-2015 David Kocher. All rights reserved. - * http://cyberduck.ch/ + * Copyright (c) 2002-2025 iterate GmbH. All rights reserved. + * https://cyberduck.io/ * * This program is free software; you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by @@ -13,11 +13,10 @@ * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * GNU General Public License for more details. - * - * Bug fixes, suggestions and comments should be sent to feedback@cyberduck.ch */ import ch.cyberduck.core.AttributedList; +import ch.cyberduck.core.DefaultIOExceptionMappingService; import ch.cyberduck.core.ListProgressListener; import ch.cyberduck.core.ListService; import ch.cyberduck.core.Path; @@ -25,18 +24,15 @@ import ch.cyberduck.core.PathNormalizer; import ch.cyberduck.core.exception.BackgroundException; import ch.cyberduck.core.exception.NotfoundException; -import ch.cyberduck.core.io.Checksum; -import org.apache.commons.codec.binary.Base64; -import org.apache.commons.codec.binary.Hex; -import org.apache.commons.io.filefilter.TrueFileFilter; import org.apache.commons.lang3.StringUtils; -import org.irods.jargon.core.exception.JargonException; -import org.irods.jargon.core.pub.IRODSFileSystemAO; -import org.irods.jargon.core.pub.domain.ObjStat; -import org.irods.jargon.core.pub.io.IRODSFile; +import org.irods.irods4j.high_level.connection.IRODSConnection; +import org.irods.irods4j.high_level.vfs.CollectionEntry; +import org.irods.irods4j.high_level.vfs.IRODSCollectionIterator; +import org.irods.irods4j.high_level.vfs.IRODSFilesystem; +import org.irods.irods4j.low_level.api.IRODSException; -import java.io.File; +import java.io.IOException; import java.util.EnumSet; public class IRODSListService implements ListService { @@ -50,34 +46,48 @@ public IRODSListService(IRODSSession session) { @Override public AttributedList list(final Path directory, final ListProgressListener listener) throws BackgroundException { try { - final AttributedList children = new AttributedList(); - final IRODSFileSystemAO fs = session.getClient(); - final IRODSFile f = fs.getIRODSFileFactory().instanceIRODSFile(directory.getAbsolute()); - if(!f.exists()) { - throw new NotfoundException(directory.getAbsolute()); + final IRODSConnection conn = session.getClient(); + + String logicalPath = directory.getAbsolute(); + if(!IRODSFilesystem.exists(conn.getRcComm(), logicalPath)) { + throw new NotfoundException(logicalPath); } - for(File file : fs.getListInDirWithFileFilter(f, TrueFileFilter.TRUE)) { - final String normalized = PathNormalizer.normalize(file.getAbsolutePath(), true); + + final AttributedList children = new AttributedList(); + + for(CollectionEntry entry : new IRODSCollectionIterator(conn.getRcComm(), logicalPath)) { + final String normalized = PathNormalizer.normalize(entry.path(), true); if(StringUtils.equals(normalized, directory.getAbsolute())) { continue; } - final PathAttributes attributes = new PathAttributes(); - final ObjStat stats = fs.getObjStat(file.getAbsolutePath()); - attributes.setModificationDate(stats.getModifiedAt().getTime()); - attributes.setCreationDate(stats.getCreatedAt().getTime()); - attributes.setSize(stats.getObjSize()); - attributes.setChecksum(Checksum.parse(Hex.encodeHexString(Base64.decodeBase64(stats.getChecksum())))); - attributes.setOwner(stats.getOwnerName()); - attributes.setGroup(stats.getOwnerZone()); - children.add(new Path(directory, PathNormalizer.name(normalized), - file.isDirectory() ? EnumSet.of(Path.Type.directory) : EnumSet.of(Path.Type.file), - attributes)); + + PathAttributes attrs = new PathAttributes(); + attrs.setCreationDate(entry.createdAt() * 1000L); + attrs.setModificationDate(entry.modifiedAt() * 1000L); + + EnumSet type = EnumSet.of(Path.Type.file); + + if(entry.isCollection()) { + attrs.setDirectoryId(entry.id()); + type = EnumSet.of(Path.Type.directory); + } + else if(entry.isDataObject()) { + attrs.setFileId(entry.id()); + attrs.setSize(entry.dataSize()); + attrs.setChecksum(IRODSChecksumUtils.toChecksum(entry.checksum())); + } + + children.add(new Path(directory, PathNormalizer.name(normalized), type, attrs)); listener.chunk(directory, children); } + return children; } - catch(JargonException e) { - throw new IRODSExceptionMappingService().map("Listing directory {0} failed", e, directory); + catch(IRODSException e) { + throw new IRODSExceptionMappingService().map("Listing {0} failed", e, directory); + } + catch(IOException e) { + throw new DefaultIOExceptionMappingService().map("Listing {0} failed", e, directory); } } } diff --git a/irods/src/main/java/ch/cyberduck/core/irods/IRODSMoveFeature.java b/irods/src/main/java/ch/cyberduck/core/irods/IRODSMoveFeature.java index 67c70daee15..5bdac39520a 100644 --- a/irods/src/main/java/ch/cyberduck/core/irods/IRODSMoveFeature.java +++ b/irods/src/main/java/ch/cyberduck/core/irods/IRODSMoveFeature.java @@ -1,8 +1,8 @@ package ch.cyberduck.core.irods; /* - * Copyright (c) 2002-2015 David Kocher. All rights reserved. - * http://cyberduck.ch/ + * Copyright (c) 2002-2025 iterate GmbH. All rights reserved. + * https://cyberduck.io/ * * This program is free software; you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by @@ -13,11 +13,10 @@ * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * GNU General Public License for more details. - * - * Bug fixes, suggestions and comments should be sent to feedback@cyberduck.ch */ import ch.cyberduck.core.ConnectionCallback; +import ch.cyberduck.core.DefaultIOExceptionMappingService; import ch.cyberduck.core.Path; import ch.cyberduck.core.exception.BackgroundException; import ch.cyberduck.core.exception.NotfoundException; @@ -25,10 +24,11 @@ import ch.cyberduck.core.features.Move; import ch.cyberduck.core.transfer.TransferStatus; -import org.irods.jargon.core.exception.JargonException; -import org.irods.jargon.core.pub.IRODSFileSystemAO; -import org.irods.jargon.core.pub.io.IRODSFile; +import org.irods.irods4j.high_level.connection.IRODSConnection; +import org.irods.irods4j.high_level.vfs.IRODSFilesystem; +import org.irods.irods4j.low_level.api.IRODSException; +import java.io.IOException; import java.util.Collections; import java.util.EnumSet; @@ -43,23 +43,25 @@ public IRODSMoveFeature(IRODSSession session) { } @Override - public Path move(final Path file, final Path renamed, final TransferStatus status, final Delete.Callback callback, final ConnectionCallback connectionCallback) throws BackgroundException { + public Path move(final Path file, final Path renamed, final TransferStatus status, + final Delete.Callback callback, final ConnectionCallback connectionCallback) throws BackgroundException { try { - final IRODSFileSystemAO fs = session.getClient(); - final IRODSFile s = fs.getIRODSFileFactory().instanceIRODSFile(file.getAbsolute()); - if(!s.exists()) { - throw new NotfoundException(String.format("%s doesn't exist", file.getAbsolute())); + final IRODSConnection conn = session.getClient(); + if(!IRODSFilesystem.exists(conn.getRcComm(), file.getAbsolute())) { + throw new NotfoundException(String.format("[%s] doesn't exist", file.getAbsolute())); } if(status.isExists()) { delete.delete(Collections.singletonMap(renamed, status), connectionCallback, callback); } - final IRODSFile d = fs.getIRODSFileFactory().instanceIRODSFile(renamed.getAbsolute()); - s.renameTo(d); + IRODSFilesystem.rename(conn.getRcComm(), file.getAbsolute(), renamed.getAbsolute()); return renamed; } - catch(JargonException e) { + catch(IRODSException e) { throw new IRODSExceptionMappingService().map("Cannot rename {0}", e, file); } + catch(IOException e) { + throw new DefaultIOExceptionMappingService().map("Cannot rename {0}", e, file); + } } @Override diff --git a/irods/src/main/java/ch/cyberduck/core/irods/IRODSProtocol.java b/irods/src/main/java/ch/cyberduck/core/irods/IRODSProtocol.java index c8e48bdea4e..8e76eafd3f5 100644 --- a/irods/src/main/java/ch/cyberduck/core/irods/IRODSProtocol.java +++ b/irods/src/main/java/ch/cyberduck/core/irods/IRODSProtocol.java @@ -1,8 +1,8 @@ package ch.cyberduck.core.irods; /* - * Copyright (c) 2002-2015 David Kocher. All rights reserved. - * http://cyberduck.ch/ + * Copyright (c) 2002-2025 iterate GmbH. All rights reserved. + * https://cyberduck.io/ * * This program is free software; you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by @@ -13,8 +13,6 @@ * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * GNU General Public License for more details. - * - * Bug fixes, suggestions and comments should be sent to feedback@cyberduck.ch */ import ch.cyberduck.core.AbstractProtocol; @@ -26,9 +24,26 @@ import com.google.auto.service.AutoService; +import java.util.HashMap; +import java.util.Map; + @AutoService(Protocol.class) public final class IRODSProtocol extends AbstractProtocol { + public static final String DESTINATION_RESOURCE = "Destination Resource"; + public static final String DELETE_OBJECTS_PERMANTENTLY = "Delete Objects Permanently"; + public static final String CLIENT_SERVER_NEGOTIATION = "Client Server Negotiation"; + public static final String TLS_PROTOCOL = "TLS Protocol"; + public static final String TLS_TRUSTSTORE = "TLS Truststore"; + public static final String TLS_TRUSTSTORE_PASSWORD = "TLS Truststore Password"; + public static final String ENCRYPTION_ALGORITHM = "Encryption Algorithm"; + public static final String ENCRYPTION_KEY_SIZE = "Encryption Key Size"; + public static final String ENCRYPTION_SALT_SIZE = "Encryption Salt Size"; + public static final String ENCRYPTION_HASH_ROUNDS = "Encryption Hash Rounds"; + public static final String PARALLEL_TRANSFER_THRESHOLD = "Parallel Transfer Threshold"; + public static final String PARALLEL_TRANSFER_CONNECTIONS = "Parallel Transfer Connections"; + public static final String PARALLEL_TRANSFER_BUFFER_SIZE = "Parallel Transfer Buffer Size"; + @Override public String getIdentifier() { return this.getScheme().name(); @@ -68,4 +83,22 @@ public String getPrefix() { public VersioningMode getVersioningMode() { return VersioningMode.none; } + + @Override + public Map getProperties() { + final Map props = new HashMap<>(); + props.put(DELETE_OBJECTS_PERMANTENTLY, "no"); + props.put(CLIENT_SERVER_NEGOTIATION, "CS_NEG_REFUSE"); + props.put(TLS_PROTOCOL, "TLSv1.2"); + props.put(TLS_TRUSTSTORE, "NOT SET"); + props.put(TLS_TRUSTSTORE_PASSWORD, "NOT SET"); + props.put(ENCRYPTION_ALGORITHM, "AES-256-CBC"); + props.put(ENCRYPTION_KEY_SIZE, "32"); + props.put(ENCRYPTION_SALT_SIZE, "8"); + props.put(ENCRYPTION_HASH_ROUNDS, "16"); + props.put(PARALLEL_TRANSFER_THRESHOLD, "33554432"); // 32MB + props.put(PARALLEL_TRANSFER_CONNECTIONS, "3"); + props.put(PARALLEL_TRANSFER_BUFFER_SIZE, "4194304"); // 4MB + return props; + } } diff --git a/irods/src/main/java/ch/cyberduck/core/irods/IRODSReadFeature.java b/irods/src/main/java/ch/cyberduck/core/irods/IRODSReadFeature.java index 636cf788056..229d1008dea 100644 --- a/irods/src/main/java/ch/cyberduck/core/irods/IRODSReadFeature.java +++ b/irods/src/main/java/ch/cyberduck/core/irods/IRODSReadFeature.java @@ -1,8 +1,8 @@ package ch.cyberduck.core.irods; /* - * Copyright (c) 2002-2015 David Kocher. All rights reserved. - * http://cyberduck.ch/ + * Copyright (c) 2002-2025 iterate GmbH. All rights reserved. + * https://cyberduck.io/ * * This program is free software; you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by @@ -13,26 +13,22 @@ * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * GNU General Public License for more details. - * - * Bug fixes, suggestions and comments should be sent to feedback@cyberduck.ch */ import ch.cyberduck.core.ConnectionCallback; +import ch.cyberduck.core.DefaultIOExceptionMappingService; import ch.cyberduck.core.Path; import ch.cyberduck.core.exception.BackgroundException; import ch.cyberduck.core.exception.NotfoundException; import ch.cyberduck.core.features.Read; -import ch.cyberduck.core.io.StreamCopier; import ch.cyberduck.core.transfer.TransferStatus; -import ch.cyberduck.core.worker.DefaultExceptionMappingService; -import org.irods.jargon.core.exception.JargonException; -import org.irods.jargon.core.exception.JargonRuntimeException; -import org.irods.jargon.core.pub.IRODSFileSystemAO; -import org.irods.jargon.core.pub.io.IRODSFile; -import org.irods.jargon.core.pub.io.IRODSFileFactory; -import org.irods.jargon.core.pub.io.PackingIrodsInputStream; +import org.irods.irods4j.high_level.io.IRODSDataObjectInputStream; +import org.irods.irods4j.high_level.vfs.IRODSFilesystem; +import org.irods.irods4j.low_level.api.IRODSApi.RcComm; +import org.irods.irods4j.low_level.api.IRODSException; +import java.io.IOException; import java.io.InputStream; public class IRODSReadFeature implements Read { @@ -46,30 +42,31 @@ public IRODSReadFeature(IRODSSession session) { @Override public InputStream read(final Path file, final TransferStatus status, final ConnectionCallback callback) throws BackgroundException { try { - try { - final IRODSFileSystemAO fs = session.getClient(); - final IRODSFileFactory factory = fs.getIRODSFileFactory(); - final IRODSFile f = factory.instanceIRODSFile(file.getAbsolute()); - if(f.exists()) { - final InputStream in = new PackingIrodsInputStream(factory.instanceIRODSFileInputStream(f)); - if(status.isAppend()) { - return StreamCopier.skip(in, status.getOffset()); - } - return in; - } - else { - throw new NotfoundException(file.getAbsolute()); - } + final RcComm rcComm = session.getClient().getRcComm(); + final String logicalPath = file.getAbsolute(); // e.g. /tempZone/home/rods/data_object.txt + + if(!IRODSFilesystem.exists(rcComm, logicalPath)) { + throw new NotfoundException(logicalPath); } - catch(JargonRuntimeException e) { - if(e.getCause() instanceof JargonException) { - throw (JargonException) e.getCause(); - } - throw new DefaultExceptionMappingService().map(e); + + IRODSDataObjectInputStream in = new IRODSDataObjectInputStream(rcComm, logicalPath); + + if(status.isAppend() && status.getOffset() > 0) { + IRODSStreamUtils.seek(in, status.getOffset()); } + + return in; } - catch(JargonException e) { - throw new IRODSExceptionMappingService().map("Download {0} failed", e, file); + catch(IRODSException e) { + throw new IRODSExceptionMappingService().map("Download of {0} failed", e, file); } + catch(IOException e) { + throw new DefaultIOExceptionMappingService().map("Download of {0} failed", e, file); + } + } + + @Override + public boolean offset(final Path file) { + return true; } } diff --git a/irods/src/main/java/ch/cyberduck/core/irods/IRODSSession.java b/irods/src/main/java/ch/cyberduck/core/irods/IRODSSession.java index fa8303775b2..f550a00b486 100644 --- a/irods/src/main/java/ch/cyberduck/core/irods/IRODSSession.java +++ b/irods/src/main/java/ch/cyberduck/core/irods/IRODSSession.java @@ -1,8 +1,8 @@ package ch.cyberduck.core.irods; /* - * Copyright (c) 2002-2015 David Kocher. All rights reserved. - * http://cyberduck.ch/ + * Copyright (c) 2002-2025 iterate GmbH. All rights reserved. + * https://cyberduck.io/ * * This program is free software; you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by @@ -13,19 +13,16 @@ * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * GNU General Public License for more details. - * - * Bug fixes, suggestions and comments should be sent to feedback@cyberduck.ch */ import ch.cyberduck.core.BookmarkNameProvider; -import ch.cyberduck.core.ConnectionTimeoutFactory; import ch.cyberduck.core.Credentials; import ch.cyberduck.core.Host; import ch.cyberduck.core.HostKeyCallback; import ch.cyberduck.core.ListService; import ch.cyberduck.core.LocaleFactory; import ch.cyberduck.core.LoginCallback; -import ch.cyberduck.core.URIEncoder; +import ch.cyberduck.core.PreferencesUseragentProvider; import ch.cyberduck.core.exception.BackgroundException; import ch.cyberduck.core.exception.LoginFailureException; import ch.cyberduck.core.features.AttributesFinder; @@ -36,7 +33,9 @@ import ch.cyberduck.core.features.Home; import ch.cyberduck.core.features.Move; import ch.cyberduck.core.features.Read; +import ch.cyberduck.core.features.Timestamp; import ch.cyberduck.core.features.Touch; +import ch.cyberduck.core.features.Upload; import ch.cyberduck.core.features.Write; import ch.cyberduck.core.proxy.ProxyFinder; import ch.cyberduck.core.shared.DefaultPathHomeFeature; @@ -49,23 +48,24 @@ import ch.cyberduck.core.ssl.X509TrustManager; import ch.cyberduck.core.threading.CancelCallback; +import ch.cyberduck.core.worker.DefaultExceptionMappingService; + import org.apache.commons.lang3.StringUtils; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; -import org.irods.jargon.core.connection.AuthScheme; -import org.irods.jargon.core.connection.IRODSAccount; -import org.irods.jargon.core.connection.SettableJargonProperties; -import org.irods.jargon.core.connection.auth.AuthResponse; -import org.irods.jargon.core.exception.JargonException; -import org.irods.jargon.core.pub.IRODSAccessObjectFactory; -import org.irods.jargon.core.pub.IRODSFileSystem; -import org.irods.jargon.core.pub.IRODSFileSystemAO; - -import java.net.URI; -import java.net.URISyntaxException; +import org.irods.irods4j.high_level.connection.IRODSConnection; +import org.irods.irods4j.high_level.connection.QualifiedUsername; +import org.irods.irods4j.low_level.api.IRODSApi; +import org.irods.irods4j.low_level.api.IRODSException; + import java.text.MessageFormat; -public class IRODSSession extends SSLSession { +public class IRODSSession extends SSLSession { + + static { + IRODSApi.setApplicationName(new PreferencesUseragentProvider().get()); + } + private static final Logger log = LogManager.getLogger(IRODSSession.class); public IRODSSession(final Host h) { @@ -77,38 +77,37 @@ public IRODSSession(final Host h, final X509TrustManager trust, final X509KeyMan } @Override - protected IRODSFileSystemAO connect(final ProxyFinder proxy, final HostKeyCallback key, final LoginCallback prompt, final CancelCallback cancel) throws BackgroundException { + public boolean isConnected() { + return super.isConnected() && null != client && client.isConnected(); + } + + @Override + protected IRODSConnection connect(final ProxyFinder proxy, final HostKeyCallback key, final LoginCallback prompt, final CancelCallback cancel) throws BackgroundException { + final String host = this.host.getHostname(); + final int port = this.host.getPort(); + final String username = this.host.getCredentials().getUsername(); + final String zone = getRegion(); + try { - final IRODSFileSystem fs = this.configure(IRODSFileSystem.instance()); - final IRODSAccessObjectFactory factory = fs.getIRODSAccessObjectFactory(); - final String region = this.getRegion(); - final String resource = this.getResource(); - final Credentials credentials = host.getCredentials(); - try { - return factory.getIRODSFileSystemAO(new URIEncodingIRODSAccount(credentials.getUsername(), credentials.getPassword(), - new IRODSHomeFinderService(IRODSSession.this).find().getAbsolute(), region, resource)); - } - catch(IllegalArgumentException e) { - throw new LoginFailureException(e.getMessage(), e); - } + log.debug("connecting to iRODS server."); + log.debug("iRODS server: host=[{}], port=[{}], username=[{}], zone=[{}]", host, port, username, zone); + + IRODSConnection conn = new IRODSConnection(IRODSConnectionUtils.initConnectionOptions(this)); + conn.connect(host, port, new QualifiedUsername(username, zone)); + log.debug("connected to iRODS server successfully."); + + return conn; } - catch(JargonException e) { - throw new IRODSExceptionMappingService().map(e); + catch(IRODSException e) { + String msg = String.format("Could not connect to iRODS server at [%s:%d] as [%s#%s]: %s", + host, port, username, zone, e.getMessage()); + throw new IRODSExceptionMappingService().map(msg, e); + } + catch(Exception e) { + String msg = String.format("Problem connecting to iRODS server at [%s:%d] as [%s#%s]: %s", + host, port, username, zone, e.getMessage()); + throw new DefaultExceptionMappingService().map(msg, e); } - } - - protected IRODSFileSystem configure(final IRODSFileSystem client) { - final SettableJargonProperties properties = new SettableJargonProperties(client.getJargonProperties()); - properties.setEncoding(host.getEncoding()); - final int timeout = ConnectionTimeoutFactory.get(preferences).getTimeout() * 1000; - properties.setIrodsSocketTimeout(timeout); - properties.setIrodsParallelSocketTimeout(timeout); - properties.setGetBufferSize(preferences.getInteger("connection.chunksize")); - properties.setPutBufferSize(preferences.getInteger("connection.chunksize")); - log.debug("Configure client {} with properties {}", client, properties); - client.getIrodsSession().setJargonProperties(properties); - client.getIrodsSession().setX509TrustManager(trust); - return client; } protected String getRegion() { @@ -128,33 +127,40 @@ protected String getResource() { @Override public void login(final LoginCallback prompt, final CancelCallback cancel) throws BackgroundException { try { - final IRODSAccount account = client.getIRODSAccount(); + log.debug("authenticating with iRODS server."); + final Credentials credentials = host.getCredentials(); - account.setUserName(credentials.getUsername()); - account.setPassword(credentials.getPassword()); - final AuthResponse response = client.getIRODSAccessObjectFactory().authenticateIRODSAccount(account); - log.debug("Connected to {}", response.getStartupResponse()); - if(!response.isSuccessful()) { - throw new LoginFailureException(MessageFormat.format(LocaleFactory.localizedString( - "Login {0} with username and password", "Credentials"), BookmarkNameProvider.toString(host))); - } + final String password = credentials.getPassword(); + + client.authenticate(IRODSConnectionUtils.newAuthPlugin(this), password); + log.debug("authenticated with iRODS server successfully."); } - catch(JargonException e) { - throw new IRODSExceptionMappingService().map(e); + catch(Exception e) { + throw new LoginFailureException(MessageFormat.format(LocaleFactory.localizedString( + "Login {0} with username and password", "Credentials"), + BookmarkNameProvider.toString(host)), e); } } @Override - public void disconnect() throws BackgroundException { + protected void disconnect() { try { - client.getIRODSSession().closeSession(); + if(null != client) { + client.disconnect(); + client = null; + log.debug("disconnected from iRODS server."); + } } - catch(JargonException e) { - throw new IRODSExceptionMappingService().map(e); + catch(Exception e) { + log.error(e.getMessage()); } - finally { + + try { super.disconnect(); } + catch(BackgroundException e) { + log.error(e.getMessage()); + } } @Override @@ -178,8 +184,8 @@ public T _getFeature(final Class type) { if(type == Move.class) { return (T) new IRODSMoveFeature(this); } - if(type == Write.class) { - return (T) new IRODSWriteFeature(this); + if(type == Upload.class) { + return (T) new IRODSUploadFeature(this); } if(type == Touch.class) { return (T) new IRODSTouchFeature(this); @@ -193,53 +199,13 @@ public T _getFeature(final Class type) { if(type == AttributesFinder.class) { return (T) new IRODSAttributesFinderFeature(this); } - return super._getFeature(type); - } - - private final class URIEncodingIRODSAccount extends IRODSAccount { - public URIEncodingIRODSAccount(final String user, final String password, final String home, final String region, final String resource) { - super(host.getHostname(), host.getPort(), StringUtils.isBlank(user) ? StringUtils.EMPTY : user, password, home, region, resource); - this.setUserName(user); - } - - @Override - public URI toURI(final boolean includePassword) throws JargonException { - try { - return new URI(String.format("irods://%s.%s%s@%s:%d%s", - this.getUserName(), - this.getZone(), - includePassword ? String.format(":%s", this.getPassword()) : StringUtils.EMPTY, - this.getHost(), - this.getPort(), - URIEncoder.encode(this.getHomeDirectory()))); - } - catch(URISyntaxException e) { - throw new JargonException(e.getMessage()); - } + if(type == Write.class) { + return (T) new IRODSWriteFeature(this); } - - @Override - public void setUserName(final String input) { - final String user; - final AuthScheme scheme; - if(StringUtils.contains(input, ':')) { - // Support non default auth scheme (PAM) - user = StringUtils.splitPreserveAllTokens(input, ':')[1]; - // Defaults to standard if not found - scheme = AuthScheme.findTypeByString(StringUtils.splitPreserveAllTokens(input, ':')[0]); - } - else { - user = input; - if(StringUtils.isNotBlank(host.getProtocol().getAuthorization())) { - scheme = AuthScheme.findTypeByString(host.getProtocol().getAuthorization()); - } - else { - // We can default to Standard if not specified - scheme = AuthScheme.STANDARD; - } - } - super.setUserName(user); - this.setAuthenticationScheme(scheme); + if(type == Timestamp.class) { + return (T) new IRODSTimestampFeature(this); } + return super._getFeature(type); } + } diff --git a/irods/src/main/java/ch/cyberduck/core/irods/IRODSStreamUtils.java b/irods/src/main/java/ch/cyberduck/core/irods/IRODSStreamUtils.java new file mode 100644 index 00000000000..b95fcbb05fd --- /dev/null +++ b/irods/src/main/java/ch/cyberduck/core/irods/IRODSStreamUtils.java @@ -0,0 +1,74 @@ +package ch.cyberduck.core.irods; + +/* + * Copyright (c) 2002-2025 iterate GmbH. All rights reserved. + * https://cyberduck.io/ + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + */ + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.irods.irods4j.high_level.io.IRODSDataObjectInputStream; +import org.irods.irods4j.high_level.io.IRODSDataObjectOutputStream; +import org.irods.irods4j.high_level.io.IRODSDataObjectStream; +import org.irods.irods4j.low_level.api.IRODSException; + +import java.io.FileInputStream; +import java.io.FileOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.nio.channels.FileChannel; + +class IRODSStreamUtils { + + private static final Logger log = LogManager.getLogger(IRODSStreamUtils.class); + + static void seek(InputStream in, long offset) throws IRODSException, IOException { + if(in instanceof IRODSDataObjectInputStream) { + IRODSDataObjectInputStream stream = (IRODSDataObjectInputStream) in; + long totalOffset = offset; + log.debug("input stream: total offset = [{}]", totalOffset); + while(totalOffset > 0) { + long intermediateOffset = Math.min(totalOffset, Integer.MAX_VALUE); + totalOffset -= intermediateOffset; + log.debug("input stream: offsetting by [{}]. remaining offset = [{}]", intermediateOffset, totalOffset); + stream.seek((int) intermediateOffset, IRODSDataObjectStream.SeekDirection.CURRENT); + } + } + else if(in instanceof FileInputStream) { + log.debug("input stream: seeking to position [{}]", offset); + FileChannel fc = ((FileInputStream) in).getChannel().position(offset); + log.debug("input stream: position = [{}]", fc.position()); + } + } + + static void seek(OutputStream out, long offset) throws IRODSException, IOException { + if(out instanceof IRODSDataObjectOutputStream) { + IRODSDataObjectOutputStream stream = (IRODSDataObjectOutputStream) out; + long totalOffset = offset; + log.debug("output stream: total offset = [{}]", totalOffset); + while(totalOffset > 0) { + long intermediateOffset = Math.min(totalOffset, Integer.MAX_VALUE); + totalOffset -= intermediateOffset; + log.debug("output stream: offsetting by [{}]. remaining offset = [{}]", intermediateOffset, totalOffset); + stream.seek((int) intermediateOffset, IRODSDataObjectStream.SeekDirection.CURRENT); + } + } + else if(out instanceof FileOutputStream) { + log.debug("output stream: seeking to position [{}]", offset); + FileChannel fc = ((FileOutputStream) out).getChannel().position(offset); + log.debug("output stream: position = [{}]", fc.position()); + } + } + +} diff --git a/irods/src/main/java/ch/cyberduck/core/irods/IRODSTimestampFeature.java b/irods/src/main/java/ch/cyberduck/core/irods/IRODSTimestampFeature.java new file mode 100644 index 00000000000..02c90195587 --- /dev/null +++ b/irods/src/main/java/ch/cyberduck/core/irods/IRODSTimestampFeature.java @@ -0,0 +1,100 @@ +package ch.cyberduck.core.irods; + +/* + * Copyright (c) 2002-2025 iterate GmbH. All rights reserved. + * https://cyberduck.io/ + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + */ + +import ch.cyberduck.core.DefaultIOExceptionMappingService; +import ch.cyberduck.core.Path; +import ch.cyberduck.core.exception.BackgroundException; +import ch.cyberduck.core.features.Timestamp; +import ch.cyberduck.core.transfer.TransferStatus; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.irods.irods4j.high_level.catalog.IRODSQuery; +import org.irods.irods4j.high_level.connection.IRODSConnection; +import org.irods.irods4j.high_level.vfs.IRODSFilesystem; +import org.irods.irods4j.high_level.vfs.IRODSReplicas; +import org.irods.irods4j.high_level.vfs.LogicalPath; +import org.irods.irods4j.high_level.vfs.ObjectStatus; +import org.irods.irods4j.low_level.api.IRODSException; + +import java.io.IOException; +import java.util.List; +import java.util.concurrent.TimeUnit; + +public class IRODSTimestampFeature implements Timestamp { + + private static final Logger log = LogManager.getLogger(IRODSTimestampFeature.class); + + private IRODSSession session; + + public IRODSTimestampFeature(IRODSSession session) { + this.session = session; + } + + @Override + public void setTimestamp(final Path file, final TransferStatus status) throws BackgroundException { + if(status.getModified() == null) { + return; + } + + final String logicalPath = file.getAbsolute(); + final long seconds = TimeUnit.MILLISECONDS.toSeconds(status.getModified()); + log.debug("setting timestamp for [{}] to [{}] seconds (since epoch).", logicalPath, seconds); + + try { + ObjectStatus objectStatus = IRODSFilesystem.status(session.getClient().getRcComm(), logicalPath); + boolean updated = true; + + if(IRODSFilesystem.isDataObject(objectStatus)) { + long replicaNumber = getReplicaNumberOfLatestGoodReplica(logicalPath); + IRODSReplicas.lastWriteTime(session.getClient().getRcComm(), logicalPath, replicaNumber, seconds); + } + else if(IRODSFilesystem.isCollection(objectStatus)) { + IRODSFilesystem.lastWriteTime(session.getClient().getRcComm(), logicalPath, seconds); + } + else { + updated = false; + log.debug("path does not point to a data object or collection. cannot update timestamp."); + } + + if(updated) { + log.debug("timestamp set to [{}] seconds (since epoch) on [{}] successfully.", seconds, logicalPath); + } + } + catch(IRODSException e) { + throw new IRODSExceptionMappingService().map(e); + } + catch(IOException e) { + throw new DefaultIOExceptionMappingService().map(e); + } + } + + private long getReplicaNumberOfLatestGoodReplica(String logicalPath) throws IRODSException, IOException { + final IRODSConnection conn = session.getClient(); + + log.debug("getting replica number of latest (good) replica."); + String query = String.format( + "select DATA_REPL_NUM, DATA_REPL_STATUS, DATA_MODIFY_TIME where COLL_NAME = '%s' and DATA_NAME = '%s' order by DATA_REPL_STATUS desc, DATA_MODIFY_TIME desc", + LogicalPath.parentPath(logicalPath), + LogicalPath.objectName(logicalPath)); + log.debug("query = [{}]", query); + List> rows = IRODSQuery.executeGenQuery2(conn.getRcComm(), query); + + return Long.parseLong(rows.get(0).get(0)); + } + +} diff --git a/irods/src/main/java/ch/cyberduck/core/irods/IRODSTouchFeature.java b/irods/src/main/java/ch/cyberduck/core/irods/IRODSTouchFeature.java index ac159c7fdd5..608fc277e79 100644 --- a/irods/src/main/java/ch/cyberduck/core/irods/IRODSTouchFeature.java +++ b/irods/src/main/java/ch/cyberduck/core/irods/IRODSTouchFeature.java @@ -1,8 +1,8 @@ package ch.cyberduck.core.irods; /* - * Copyright (c) 2002-2015 David Kocher. All rights reserved. - * http://cyberduck.ch/ + * Copyright (c) 2002-2025 iterate GmbH. All rights reserved. + * https://cyberduck.io/ * * This program is free software; you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by @@ -13,22 +13,29 @@ * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * GNU General Public License for more details. - * - * Bug fixes, suggestions and comments should be sent to feedback@cyberduck.ch */ +import ch.cyberduck.core.DefaultIOExceptionMappingService; import ch.cyberduck.core.Path; import ch.cyberduck.core.exception.BackgroundException; import ch.cyberduck.core.features.Touch; import ch.cyberduck.core.features.Write; import ch.cyberduck.core.transfer.TransferStatus; -import org.irods.jargon.core.exception.JargonException; -import org.irods.jargon.core.packinstr.DataObjInp; -import org.irods.jargon.core.pub.IRODSFileSystemAO; +import com.fasterxml.jackson.databind.ObjectMapper; + +import org.irods.irods4j.high_level.connection.IRODSConnection; +import org.irods.irods4j.low_level.api.IRODSApi; +import org.irods.irods4j.low_level.api.IRODSException; + +import java.io.IOException; +import java.util.HashMap; +import java.util.Map; public class IRODSTouchFeature implements Touch { + private static final ObjectMapper mapper = new ObjectMapper(); + private final IRODSSession session; public IRODSTouchFeature(final IRODSSession session) { @@ -38,14 +45,30 @@ public IRODSTouchFeature(final IRODSSession session) { @Override public Path touch(final Write writer, final Path file, final TransferStatus status) throws BackgroundException { try { - final IRODSFileSystemAO fs = session.getClient(); - final int descriptor = fs.createFile(file.getAbsolute(), - DataObjInp.OpenFlags.WRITE_TRUNCATE, DataObjInp.DEFAULT_CREATE_MODE); - fs.fileClose(descriptor, false); + final IRODSConnection conn = session.getClient(); + + Map input = new HashMap<>(); + input.put("logical_path", file.getAbsolute()); + + String jsonInput = mapper.writeValueAsString(input); + + int ec = IRODSApi.rcTouch(conn.getRcComm(), jsonInput); + if(ec < 0) { + throw new IRODSException(ec, "rcTouch error"); + } + return file; } - catch(JargonException e) { + catch(IRODSException e) { throw new IRODSExceptionMappingService().map("Cannot create {0}", e, file); } + catch(IOException e) { + throw new DefaultIOExceptionMappingService().map("Cannot create {0}", e, file); + } + } + + @Override + public boolean isSupported(final Path workdir, final String filename) { + return true; } } diff --git a/irods/src/main/java/ch/cyberduck/core/irods/IRODSUploadFeature.java b/irods/src/main/java/ch/cyberduck/core/irods/IRODSUploadFeature.java index 93f4b591879..10da289e5d5 100644 --- a/irods/src/main/java/ch/cyberduck/core/irods/IRODSUploadFeature.java +++ b/irods/src/main/java/ch/cyberduck/core/irods/IRODSUploadFeature.java @@ -1,8 +1,8 @@ package ch.cyberduck.core.irods; /* - * Copyright (c) 2002-2015 David Kocher. All rights reserved. - * http://cyberduck.ch/ + * Copyright (c) 2002-2025 iterate GmbH. All rights reserved. + * https://cyberduck.io/ * * This program is free software; you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by @@ -13,45 +13,47 @@ * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * GNU General Public License for more details. - * - * Bug fixes, suggestions and comments should be sent to feedback@cyberduck.ch */ import ch.cyberduck.core.ConnectionCallback; -import ch.cyberduck.core.Host; +import ch.cyberduck.core.DefaultIOExceptionMappingService; import ch.cyberduck.core.Local; -import ch.cyberduck.core.LocaleFactory; import ch.cyberduck.core.Path; import ch.cyberduck.core.ProgressListener; import ch.cyberduck.core.exception.BackgroundException; -import ch.cyberduck.core.exception.ChecksumException; +import ch.cyberduck.core.exception.ConnectionCanceledException; import ch.cyberduck.core.features.Upload; import ch.cyberduck.core.features.Write; import ch.cyberduck.core.io.BandwidthThrottle; -import ch.cyberduck.core.io.Checksum; -import ch.cyberduck.core.io.ChecksumComputeFactory; import ch.cyberduck.core.io.StreamListener; import ch.cyberduck.core.preferences.HostPreferencesFactory; -import ch.cyberduck.core.preferences.PreferencesFactory; +import ch.cyberduck.core.preferences.PreferencesReader; +import ch.cyberduck.core.threading.ThreadPool; +import ch.cyberduck.core.threading.ThreadPoolFactory; import ch.cyberduck.core.transfer.TransferStatus; +import ch.cyberduck.core.worker.DefaultExceptionMappingService; + import org.apache.commons.lang3.StringUtils; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; -import org.irods.jargon.core.checksum.ChecksumValue; -import org.irods.jargon.core.exception.JargonException; -import org.irods.jargon.core.packinstr.TransferOptions; -import org.irods.jargon.core.pub.DataObjectChecksumUtilitiesAO; -import org.irods.jargon.core.pub.DataTransferOperations; -import org.irods.jargon.core.pub.IRODSFileSystemAO; -import org.irods.jargon.core.pub.io.IRODSFile; -import org.irods.jargon.core.transfer.DefaultTransferControlBlock; -import org.irods.jargon.core.transfer.TransferControlBlock; - -import java.io.File; -import java.text.MessageFormat; +import org.irods.irods4j.high_level.connection.IRODSConnection; +import org.irods.irods4j.high_level.connection.IRODSConnectionPool; +import org.irods.irods4j.high_level.connection.IRODSConnectionPool.PoolConnection; +import org.irods.irods4j.high_level.io.IRODSDataObjectOutputStream; +import org.irods.irods4j.high_level.io.IRODSDataObjectStream; +import org.irods.irods4j.low_level.api.IRODSException; + +import java.io.FileInputStream; +import java.io.IOException; +import java.io.InputStream; +import java.util.ArrayList; +import java.util.List; +import java.util.Optional; +import java.util.concurrent.Future; public class IRODSUploadFeature implements Upload { + private static final Logger log = LogManager.getLogger(IRODSUploadFeature.class); private final IRODSSession session; @@ -65,54 +67,285 @@ public Void upload(final Write write, final Path file, final Local local, final ProgressListener progress, final StreamListener streamListener, final TransferStatus status, final ConnectionCallback callback) throws BackgroundException { try { - final IRODSFileSystemAO fs = session.getClient(); - final IRODSFile f = fs.getIRODSFileFactory().instanceIRODSFile(file.getAbsolute()); - final TransferControlBlock block = DefaultTransferControlBlock.instance(StringUtils.EMPTY, - HostPreferencesFactory.get(session.getHost()).getInteger("connection.retry")); - final TransferOptions options = new DefaultTransferOptionsConfigurer().configure(new TransferOptions()); - if(Host.TransferType.unknown.equals(session.getHost().getTransferType())) { - options.setUseParallelTransfer(Host.TransferType.valueOf(PreferencesFactory.get().getProperty("queue.transfer.type")).equals(Host.TransferType.concurrent)); - } - else { - options.setUseParallelTransfer(session.getHost().getTransferType().equals(Host.TransferType.concurrent)); - } - block.setTransferOptions(options); - final DataTransferOperations transfer = fs.getIRODSAccessObjectFactory().getDataTransferOperations(fs.getIRODSAccount()); - transfer.putOperation(new File(local.getAbsolute()), f, new DefaultTransferStatusCallbackListener( - status, streamListener, block - ), block); - if(status.isComplete()) { - final DataObjectChecksumUtilitiesAO checksum = fs - .getIRODSAccessObjectFactory() - .getDataObjectChecksumUtilitiesAO(fs.getIRODSAccount()); - final ChecksumValue value = checksum.computeChecksumOnDataObject(f); - final Checksum fingerprint = Checksum.parse(value.getChecksumStringValue()); - if(null == fingerprint) { - log.warn("Unsupported checksum algorithm {}", value.getChecksumEncoding()); - } - else { - if(file.getType().contains(Path.Type.encrypted)) { - log.warn("Skip checksum verification for {} with client side encryption enabled", file); + final PreferencesReader preferences = HostPreferencesFactory.get(session.getHost()); + + final long fileSize = local.attributes().getSize(); + final String logicalPath = file.getAbsolute(); + final String dstRootResource = getResource(preferences).orElse(StringUtils.EMPTY); + + log.debug("status.getLength() = [{}]", status.getLength()); + log.debug("fileSize = [{}]", fileSize); + log.debug("local file = [{}]", local.getAbsolute()); + log.debug("logicalPath = [{}]", logicalPath); + log.debug("dst root resource = [{}]", dstRootResource); + + // Signals whether the completion flag should be set following the transfer. + // An upload is considered to be successful if and only if no errors occurred. + // That includes waiting for background threads to terminate and closing output + // streams. This is important for certain features (e.g. mtime preservation). + boolean setCompletionFlag = false; + + final long threshold = IRODSIntegerUtils.clamp( + preferences.getInteger(IRODSProtocol.PARALLEL_TRANSFER_THRESHOLD), 1, Integer.MAX_VALUE); + final int bufferSize = IRODSIntegerUtils.clamp( + preferences.getInteger(IRODSProtocol.PARALLEL_TRANSFER_BUFFER_SIZE), 1, (int) (128 * TransferStatus.MEGA)); + + // Transfer the bytes over multiple connections if the size of the local file + // exceeds a certain number of bytes - e.g. 32MB. + if(fileSize < threshold) { + log.debug("local file does not exceed parallel transfer threshold [{}]. performing single-threaded transfer.", threshold); + + byte[] buffer = new byte[bufferSize]; + boolean truncate = true; + boolean append = false; + + try(FileInputStream in = new FileInputStream(local.getAbsolute()); + IRODSConnection conn = IRODSConnectionUtils.newAuthenticatedConnection(session)) { + + IRODSDataObjectOutputStream out; + if(StringUtils.isNotBlank(dstRootResource)) { + out = new IRODSDataObjectOutputStream(conn.getRcComm(), logicalPath, dstRootResource, truncate, append); } else { - final Checksum expected = ChecksumComputeFactory.get(fingerprint.algorithm).compute(local.getInputStream(), new TransferStatus(status)); - if(!expected.equals(fingerprint)) { - throw new ChecksumException(MessageFormat.format(LocaleFactory.localizedString("Upload {0} failed", "Error"), file.getName()), - MessageFormat.format("Mismatch between {0} hash {1} of uploaded data and ETag {2} returned by the server", - fingerprint.algorithm.toString(), expected, fingerprint.hash)); + out = new IRODSDataObjectOutputStream(conn.getRcComm(), logicalPath, truncate, append); + } + + try { + while(true) { + try { + status.validate(); // Throws if transfer is cancelled. + } + catch(ConnectionCanceledException e) { + log.info("transfer cancelled."); + return null; + } + + int bytesRead = in.read(buffer); + if(bytesRead == -1) { + setCompletionFlag = true; + return null; + } + streamListener.recv(bytesRead); + out.write(buffer, 0, bytesRead); + streamListener.sent(bytesRead); + } + } + finally { + out.close(); + + if(setCompletionFlag) { + status.setComplete(); } } } } + + // + // The data object is larger than the threshold so use parallel transfer. + // + + log.debug("local file exceeds parallel transfer threshold [{}]. performing multi-threaded transfer.", threshold); + + final int threadCount = IRODSIntegerUtils.clamp( + preferences.getInteger(IRODSProtocol.PARALLEL_TRANSFER_CONNECTIONS), 2, 10); + log.debug("thread count = [{}]; starting thread pool.", threadCount); + final ThreadPool threadPool = ThreadPoolFactory.get("multipart-iRODS", threadCount); + + final long chunkSize = fileSize / threadCount; + final long remainingBytes = fileSize % threadCount; + log.debug("chunk size = [{}]", chunkSize); + log.debug("remaining bytes = [{}]", remainingBytes); + + final List localFileStreams = new ArrayList<>(); + final List irodsStreams = new ArrayList<>(); + + log.debug("launching connection pool with [{}] connections.", threadCount); + try(IRODSConnectionPool pool = new IRODSConnectionPool(IRODSConnectionUtils.initConnectionOptions(session), threadCount)) { + try { + status.validate(); // Throws if transfer is cancelled. + } + catch(ConnectionCanceledException e) { + log.info("transfer cancelled."); + return null; + } + + IRODSConnectionUtils.startIRODSConnectionPool(session, pool); + log.debug("connection pool started."); + + try { + String replicaToken = null; + long replicaNumber = -1; + + for(int i = 0; i < threadCount; ++i) { + // We cannot use Files.newInputStream() does not report the concrete + // type of the stream. The concrete type is needed for seek operations. + localFileStreams.add(new FileInputStream(local.getAbsolute())); + + // The pooled connection will never be returned to the pool. This is + // okay because after the transfer, no connection is reused. + PoolConnection conn = pool.getConnection(); + + if(0 == i) { + log.debug("opened primary iRODS stream."); + // The first iRODS output stream is the primary stream. The opened + // replica is always truncated upon success. + if(StringUtils.isNotBlank(dstRootResource)) { + irodsStreams.add(new IRODSDataObjectOutputStream( + conn.getRcComm(), logicalPath, dstRootResource, true, false)); + } + else { + irodsStreams.add(new IRODSDataObjectOutputStream( + conn.getRcComm(), logicalPath, true, false)); + } + replicaToken = irodsStreams.get(0).getReplicaToken(); + replicaNumber = irodsStreams.get(0).getReplicaNumber(); + log.debug("replica token = [{}]", replicaToken); + log.debug("replica number = [{}]", replicaNumber); + } + else { + log.debug("opened secondary iRODS stream."); + irodsStreams.add(new IRODSDataObjectOutputStream( + conn.getRcComm(), replicaToken, logicalPath, replicaNumber, false, false)); + } + } + + try { + status.validate(); // Throws if transfer is cancelled. + } + catch(ConnectionCanceledException e) { + log.info("transfer cancelled."); + return null; + } + + // Holds handles to tasks running on the thread pool. This allows us to wait for + // all tasks to complete before shutting down everything. + List> tasks = new ArrayList<>(); + + // Launch remaining IO tasks. + log.debug("launch parallel IO tasks."); + for(int i = 0; i < threadCount; ++i) { + tasks.add(threadPool.execute(new IRODSChunkWorker( + status, + streamListener, + localFileStreams.get(i), + irodsStreams.get(i), + i * chunkSize, + (threadCount - 1 == i) ? chunkSize + remainingBytes : chunkSize, + bufferSize + ))); + } + + setCompletionFlag = waitForTasksToComplete(tasks); + } + finally { + final boolean closedOutputStreams = closeOutputStreams(irodsStreams); + if(setCompletionFlag && closedOutputStreams) { + status.setComplete(); + } + + closeInputStreams(localFileStreams); + } + } + + log.debug("shutting down thread pool."); + threadPool.shutdown(false); + log.debug("done."); + return null; } - catch(JargonException e) { + catch(IRODSException e) { throw new IRODSExceptionMappingService().map(e); } + catch(IOException e) { + throw new DefaultIOExceptionMappingService().map(e); + } + catch(Exception e) { + throw new DefaultExceptionMappingService().map(e); + } } @Override - public Write.Append append(final Path file, final TransferStatus status) throws BackgroundException { + public Write.Append append(Path file, TransferStatus status) throws BackgroundException { return new Write.Append(status.isExists()).withStatus(status); } + + private static boolean closeOutputStreams(List streams) { + log.debug("closing output streams."); + + final IRODSDataObjectStream.OnCloseSuccess closeInstructions = new IRODSDataObjectStream.OnCloseSuccess(); + closeInstructions.updateSize = false; + closeInstructions.updateStatus = false; + closeInstructions.computeChecksum = false; + closeInstructions.sendNotifications = false; + + boolean success = true; + + for(int i = 1; i < streams.size(); ++i) { + try { + streams.get(i).close(closeInstructions); + } + catch(Exception e) { + log.error(e.getMessage()); + success = false; + } + } + + try { + streams.get(0).close(); + } + catch(Exception e) { + log.error(e.getMessage()); + success = false; + } + + return success; + } + + private static void closeInputStreams(List streams) { + log.debug("closing input streams."); + + streams.forEach(out -> { + try { + out.close(); + } + catch(Exception e) { + log.error(e.getMessage()); + } + }); + } + + private static boolean waitForTasksToComplete(List> tasks) { + log.debug("waiting for parallel IO tasks to finish."); + boolean success = true; + + for(Future task : tasks) { + try { + if(!task.get()) { + success = false; + } + } + catch(Exception e) { + success = false; + log.error(e.getMessage()); + } + } + + log.debug("parallel IO tasks have finished."); + return success; + } + + private Optional getResource(PreferencesReader prefs) { + final String resc = prefs.getProperty(IRODSProtocol.DESTINATION_RESOURCE); + if(StringUtils.isNotBlank(resc)) { + return Optional.of(resc); + } + + final String region = session.getHost().getRegion(); + int index = region.indexOf(':'); + if(index != -1 && ++index < region.length()) { + return Optional.of(region.substring(index)); + } + + return Optional.empty(); + } } diff --git a/irods/src/main/java/ch/cyberduck/core/irods/IRODSWriteFeature.java b/irods/src/main/java/ch/cyberduck/core/irods/IRODSWriteFeature.java index e567a368af5..906c56abd01 100644 --- a/irods/src/main/java/ch/cyberduck/core/irods/IRODSWriteFeature.java +++ b/irods/src/main/java/ch/cyberduck/core/irods/IRODSWriteFeature.java @@ -1,8 +1,8 @@ package ch.cyberduck.core.irods; /* - * Copyright (c) 2002-2015 David Kocher. All rights reserved. - * http://cyberduck.ch/ + * Copyright (c) 2002-2025 iterate GmbH. All rights reserved. + * https://cyberduck.io/ * * This program is free software; you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by @@ -13,28 +13,30 @@ * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * GNU General Public License for more details. - * - * Bug fixes, suggestions and comments should be sent to feedback@cyberduck.ch */ import ch.cyberduck.core.ConnectionCallback; +import ch.cyberduck.core.DefaultIOExceptionMappingService; import ch.cyberduck.core.Path; import ch.cyberduck.core.exception.BackgroundException; import ch.cyberduck.core.features.Write; import ch.cyberduck.core.io.StatusOutputStream; import ch.cyberduck.core.io.VoidStatusOutputStream; import ch.cyberduck.core.transfer.TransferStatus; -import ch.cyberduck.core.worker.DefaultExceptionMappingService; -import org.irods.jargon.core.exception.JargonException; -import org.irods.jargon.core.exception.JargonRuntimeException; -import org.irods.jargon.core.packinstr.DataObjInp; -import org.irods.jargon.core.pub.IRODSFileSystemAO; -import org.irods.jargon.core.pub.io.IRODSFileOutputStream; -import org.irods.jargon.core.pub.io.PackingIrodsOutputStream; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.irods.irods4j.high_level.connection.IRODSConnection; +import org.irods.irods4j.high_level.io.IRODSDataObjectOutputStream; +import org.irods.irods4j.low_level.api.IRODSException; + +import java.io.IOException; +import java.io.OutputStream; public class IRODSWriteFeature implements Write { + private static final Logger log = LogManager.getLogger(IRODSWriteFeature.class); + private final IRODSSession session; public IRODSWriteFeature(IRODSSession session) { @@ -44,21 +46,17 @@ public IRODSWriteFeature(IRODSSession session) { @Override public StatusOutputStream write(final Path file, final TransferStatus status, final ConnectionCallback callback) throws BackgroundException { try { - try { - final IRODSFileSystemAO fs = session.getClient(); - final IRODSFileOutputStream out = fs.getIRODSFileFactory().instanceIRODSFileOutputStream( - file.getAbsolute(), status.isAppend() ? DataObjInp.OpenFlags.READ_WRITE : DataObjInp.OpenFlags.WRITE_TRUNCATE); - return new VoidStatusOutputStream(new PackingIrodsOutputStream(out)); - } - catch(JargonRuntimeException e) { - if(e.getCause() instanceof JargonException) { - throw (JargonException) e.getCause(); - } - throw new DefaultExceptionMappingService().map(e); - } + final IRODSConnection conn = session.getClient(); + boolean append = status.isAppend(); + boolean truncate = !append; + final OutputStream out = new IRODSDataObjectOutputStream(conn.getRcComm(), file.getAbsolute(), truncate, append); + return new VoidStatusOutputStream(out); } - catch(JargonException e) { + catch(IRODSException e) { throw new IRODSExceptionMappingService().map("Uploading {0} failed", e, file); } + catch(IOException e) { + throw new DefaultIOExceptionMappingService().map("Uploading {0} failed", e, file); + } } } diff --git a/irods/src/test/java/ch/cyberduck/core/irods/IRODSAttributesFinderFeatureTest.java b/irods/src/test/java/ch/cyberduck/core/irods/IRODSAttributesFinderFeatureTest.java index c5e479a359e..58817f36df9 100644 --- a/irods/src/test/java/ch/cyberduck/core/irods/IRODSAttributesFinderFeatureTest.java +++ b/irods/src/test/java/ch/cyberduck/core/irods/IRODSAttributesFinderFeatureTest.java @@ -29,8 +29,7 @@ import ch.cyberduck.core.proxy.DisabledProxyFinder; import ch.cyberduck.core.serializer.impl.dd.ProfilePlistReader; import ch.cyberduck.core.transfer.TransferStatus; -import ch.cyberduck.test.IntegrationTest; -import ch.cyberduck.test.VaultTest; +import ch.cyberduck.test.TestcontainerTest; import org.junit.Test; import org.junit.experimental.categories.Category; @@ -43,14 +42,14 @@ import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; -@Category(IntegrationTest.class) -public class IRODSAttributesFinderFeatureTest extends VaultTest { +@Category(TestcontainerTest.class) +public class IRODSAttributesFinderFeatureTest extends IRODSDockerComposeManager { @Test(expected = NotfoundException.class) public void testFindNotFound() throws Exception { final ProtocolFactory factory = new ProtocolFactory(new HashSet<>(Collections.singleton(new IRODSProtocol()))); final Profile profile = new ProfilePlistReader(factory).read( - this.getClass().getResourceAsStream("/iRODS (iPlant Collaborative).cyberduckprofile")); + this.getClass().getResourceAsStream("/iRODS.cyberduckprofile")); final Host host = new Host(profile, profile.getDefaultHostname(), new Credentials( PROPERTIES.get("irods.key"), PROPERTIES.get("irods.secret") )); @@ -64,7 +63,7 @@ public void testFindNotFound() throws Exception { public void testFind() throws Exception { final ProtocolFactory factory = new ProtocolFactory(new HashSet<>(Collections.singleton(new IRODSProtocol()))); final Profile profile = new ProfilePlistReader(factory).read( - this.getClass().getResourceAsStream("/iRODS (iPlant Collaborative).cyberduckprofile")); + this.getClass().getResourceAsStream("/iRODS.cyberduckprofile")); final Host host = new Host(profile, profile.getDefaultHostname(), new Credentials( PROPERTIES.get("irods.key"), PROPERTIES.get("irods.secret") )); @@ -81,8 +80,6 @@ public void testFind() throws Exception { assertEquals(folderTimestamp, f.find(folder).getModificationDate()); final PathAttributes attributes = f.find(test); assertEquals(0L, attributes.getSize()); - assertEquals("iterate", attributes.getOwner()); - assertEquals("iplant", attributes.getGroup()); new IRODSDeleteFeature(session).delete(Collections.singletonList(test), new DisabledLoginCallback(), new Delete.DisabledCallback()); assertFalse(new IRODSFindFeature(session).find(test)); session.close(); diff --git a/irods/src/test/java/ch/cyberduck/core/irods/IRODSChecksumUtilsTest.java b/irods/src/test/java/ch/cyberduck/core/irods/IRODSChecksumUtilsTest.java new file mode 100644 index 00000000000..154adf205a5 --- /dev/null +++ b/irods/src/test/java/ch/cyberduck/core/irods/IRODSChecksumUtilsTest.java @@ -0,0 +1,49 @@ +package ch.cyberduck.core.irods; + +/* + * Copyright (c) 2002-2015 David Kocher. All rights reserved. + * http://cyberduck.ch/ + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * Bug fixes, suggestions and comments should be sent to feedback@cyberduck.ch + */ + +import ch.cyberduck.core.io.Checksum; + +import ch.cyberduck.core.io.HashAlgorithm; + +import org.apache.commons.codec.binary.Base64; +import org.apache.commons.codec.binary.Hex; +import org.junit.Test; + +import static org.junit.Assert.assertEquals; + +public class IRODSChecksumUtilsTest { + + @Test + public void testInvalidInputs() { + assertEquals(Checksum.NONE, IRODSChecksumUtils.toChecksum(null)); + assertEquals(Checksum.NONE, IRODSChecksumUtils.toChecksum("")); + assertEquals(Checksum.NONE, IRODSChecksumUtils.toChecksum("no_colon")); + assertEquals(Checksum.NONE, IRODSChecksumUtils.toChecksum("sha2:")); + } + + @Test + public void testValidChecksum() { + final String irodsChecksum = "sha2:Qu9ZCAIUVS3AR4gaILMZevr5PK7XAO9vqlhSF0u+ha0="; + final String checksumNoPrefix = irodsChecksum.substring(irodsChecksum.indexOf(':') + 1); + final String hexEncodedChecksum = Hex.encodeHexString(Base64.decodeBase64(checksumNoPrefix)); + final Checksum expectedChecksum = Checksum.parse(hexEncodedChecksum); + assertEquals(expectedChecksum, IRODSChecksumUtils.toChecksum(irodsChecksum)); + assertEquals(HashAlgorithm.sha256, expectedChecksum.algorithm); + } +} diff --git a/irods/src/test/java/ch/cyberduck/core/irods/IRODSCopyFeatureTest.java b/irods/src/test/java/ch/cyberduck/core/irods/IRODSCopyFeatureTest.java index 6db993b2ab8..5f92bc8358a 100644 --- a/irods/src/test/java/ch/cyberduck/core/irods/IRODSCopyFeatureTest.java +++ b/irods/src/test/java/ch/cyberduck/core/irods/IRODSCopyFeatureTest.java @@ -31,8 +31,7 @@ import ch.cyberduck.core.proxy.DisabledProxyFinder; import ch.cyberduck.core.serializer.impl.dd.ProfilePlistReader; import ch.cyberduck.core.transfer.TransferStatus; -import ch.cyberduck.test.IntegrationTest; -import ch.cyberduck.test.VaultTest; +import ch.cyberduck.test.TestcontainerTest; import org.junit.Test; import org.junit.experimental.categories.Category; @@ -45,14 +44,14 @@ import static org.junit.Assert.assertTrue; -@Category(IntegrationTest.class) -public class IRODSCopyFeatureTest extends VaultTest { +@Category(TestcontainerTest.class) +public class IRODSCopyFeatureTest extends IRODSDockerComposeManager { @Test public void testCopy() throws Exception { final ProtocolFactory factory = new ProtocolFactory(new HashSet<>(Collections.singleton(new IRODSProtocol()))); final Profile profile = new ProfilePlistReader(factory).read( - this.getClass().getResourceAsStream("/iRODS (iPlant Collaborative).cyberduckprofile")); + this.getClass().getResourceAsStream("/iRODS.cyberduckprofile")); final Host host = new Host(profile, profile.getDefaultHostname(), new Credentials( PROPERTIES.get("irods.key"), PROPERTIES.get("irods.secret") )); diff --git a/irods/src/test/java/ch/cyberduck/core/irods/IRODSDeleteFeatureTest.java b/irods/src/test/java/ch/cyberduck/core/irods/IRODSDeleteFeatureTest.java index 2936c7c3df5..8ce521c2c94 100644 --- a/irods/src/test/java/ch/cyberduck/core/irods/IRODSDeleteFeatureTest.java +++ b/irods/src/test/java/ch/cyberduck/core/irods/IRODSDeleteFeatureTest.java @@ -30,8 +30,7 @@ import ch.cyberduck.core.proxy.DisabledProxyFinder; import ch.cyberduck.core.serializer.impl.dd.ProfilePlistReader; import ch.cyberduck.core.transfer.TransferStatus; -import ch.cyberduck.test.IntegrationTest; -import ch.cyberduck.test.VaultTest; +import ch.cyberduck.test.TestcontainerTest; import org.junit.Test; import org.junit.experimental.categories.Category; @@ -44,14 +43,14 @@ import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertTrue; -@Category(IntegrationTest.class) -public class IRODSDeleteFeatureTest extends VaultTest { +@Category(TestcontainerTest.class) +public class IRODSDeleteFeatureTest extends IRODSDockerComposeManager { @Test public void testDeleteDirectory() throws Exception { final ProtocolFactory factory = new ProtocolFactory(new HashSet<>(Collections.singleton(new IRODSProtocol()))); final Profile profile = new ProfilePlistReader(factory).read( - this.getClass().getResourceAsStream("/iRODS (iPlant Collaborative).cyberduckprofile")); + this.getClass().getResourceAsStream("/iRODS.cyberduckprofile")); final Host host = new Host(profile, profile.getDefaultHostname(), new Credentials( PROPERTIES.get("irods.key"), PROPERTIES.get("irods.secret") )); @@ -76,7 +75,7 @@ public void testDeleteDirectory() throws Exception { public void testDeleteNotFound() throws Exception { final ProtocolFactory factory = new ProtocolFactory(new HashSet<>(Collections.singleton(new IRODSProtocol()))); final Profile profile = new ProfilePlistReader(factory).read( - this.getClass().getResourceAsStream("/iRODS (iPlant Collaborative).cyberduckprofile")); + this.getClass().getResourceAsStream("/iRODS.cyberduckprofile")); final Host host = new Host(profile, profile.getDefaultHostname(), new Credentials( PROPERTIES.get("irods.key"), PROPERTIES.get("irods.secret") )); diff --git a/irods/src/test/java/ch/cyberduck/core/irods/IRODSDirectoryFeatureTest.java b/irods/src/test/java/ch/cyberduck/core/irods/IRODSDirectoryFeatureTest.java index 9b4e7168862..35a0e7c051b 100644 --- a/irods/src/test/java/ch/cyberduck/core/irods/IRODSDirectoryFeatureTest.java +++ b/irods/src/test/java/ch/cyberduck/core/irods/IRODSDirectoryFeatureTest.java @@ -31,8 +31,7 @@ import ch.cyberduck.core.proxy.DisabledProxyFinder; import ch.cyberduck.core.serializer.impl.dd.ProfilePlistReader; import ch.cyberduck.core.transfer.TransferStatus; -import ch.cyberduck.test.IntegrationTest; -import ch.cyberduck.test.VaultTest; +import ch.cyberduck.test.TestcontainerTest; import org.junit.Test; import org.junit.experimental.categories.Category; @@ -45,14 +44,14 @@ import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertTrue; -@Category(IntegrationTest.class) -public class IRODSDirectoryFeatureTest extends VaultTest { +@Category(TestcontainerTest.class) +public class IRODSDirectoryFeatureTest extends IRODSDockerComposeManager { @Test public void testMakeDirectory() throws Exception { final ProtocolFactory factory = new ProtocolFactory(new HashSet<>(Collections.singleton(new IRODSProtocol()))); final Profile profile = new ProfilePlistReader(factory).read( - this.getClass().getResourceAsStream("/iRODS (iPlant Collaborative).cyberduckprofile")); + this.getClass().getResourceAsStream("/iRODS.cyberduckprofile")); final Host host = new Host(profile, profile.getDefaultHostname(), new Credentials( PROPERTIES.get("irods.key"), PROPERTIES.get("irods.secret") )); diff --git a/irods/src/test/java/ch/cyberduck/core/irods/IRODSDockerComposeManager.java b/irods/src/test/java/ch/cyberduck/core/irods/IRODSDockerComposeManager.java new file mode 100644 index 00000000000..cf20df441d4 --- /dev/null +++ b/irods/src/test/java/ch/cyberduck/core/irods/IRODSDockerComposeManager.java @@ -0,0 +1,50 @@ +package ch.cyberduck.core.irods; + +/* + * Copyright (c) 2002-2025 iterate GmbH. All rights reserved. + * https://cyberduck.io/ + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + */ + +import ch.cyberduck.test.TestcontainerTest; + +import org.junit.experimental.categories.Category; +import org.testcontainers.containers.ComposeContainer; +import org.testcontainers.containers.wait.strategy.Wait; + +import java.io.File; +import java.util.HashMap; +import java.util.Map; + +@Category(TestcontainerTest.class) +public abstract class IRODSDockerComposeManager { + + private static final ComposeContainer container; + + // Launch the docker compose project once and run all tests against that environment. + // Ryuk will clean it up at the end of the run. + static { + container = new ComposeContainer( + new File(IRODSDockerComposeManager.class.getResource("/docker/docker-compose.yml").getFile())) + .withPull(false) + .withLocalCompose(true) + .withExposedService("irods-catalog-provider-1", 1247, Wait.forLogMessage(".*\"log_message\":\"Initializing delay server.\".*", 1)); + + container.start(); + } + + protected static final Map PROPERTIES = new HashMap() {{ + put("irods.key", "rods"); + put("irods.secret", "rods"); + }}; + +} diff --git a/irods/src/test/java/ch/cyberduck/core/irods/IRODSExceptionMappingServiceTest.java b/irods/src/test/java/ch/cyberduck/core/irods/IRODSExceptionMappingServiceTest.java index c7b0664aa76..61951917a12 100644 --- a/irods/src/test/java/ch/cyberduck/core/irods/IRODSExceptionMappingServiceTest.java +++ b/irods/src/test/java/ch/cyberduck/core/irods/IRODSExceptionMappingServiceTest.java @@ -18,12 +18,12 @@ */ import ch.cyberduck.core.exception.AccessDeniedException; + import ch.cyberduck.core.exception.LoginFailureException; import ch.cyberduck.core.exception.NotfoundException; -import org.irods.jargon.core.exception.AuthenticationException; -import org.irods.jargon.core.exception.CatNoAccessException; -import org.irods.jargon.core.exception.FileNotFoundException; +import org.irods.irods4j.low_level.api.IRODSErrorCodes; +import org.irods.irods4j.low_level.api.IRODSException; import org.junit.Test; import static org.junit.Assert.assertTrue; @@ -32,8 +32,9 @@ public class IRODSExceptionMappingServiceTest { @Test public void testMap() { - assertTrue(new IRODSExceptionMappingService().map(new CatNoAccessException("no access")) instanceof AccessDeniedException); - assertTrue(new IRODSExceptionMappingService().map(new FileNotFoundException("no file")) instanceof NotfoundException); - assertTrue(new IRODSExceptionMappingService().map(new AuthenticationException("no user")) instanceof LoginFailureException); + final IRODSExceptionMappingService mappingService = new IRODSExceptionMappingService(); + assertTrue(mappingService.map(new IRODSException(IRODSErrorCodes.CAT_NO_ACCESS_PERMISSION, "no access")) instanceof AccessDeniedException); + assertTrue(mappingService.map(new IRODSException(IRODSErrorCodes.CAT_NO_ROWS_FOUND, "no file")) instanceof NotfoundException); + assertTrue(mappingService.map(new IRODSException(IRODSErrorCodes.AUTHENTICATION_ERROR, "no user")) instanceof LoginFailureException); } } diff --git a/irods/src/test/java/ch/cyberduck/core/irods/IRODSFindFeatureTest.java b/irods/src/test/java/ch/cyberduck/core/irods/IRODSFindFeatureTest.java index a1e9fb93142..bbdf9d67836 100644 --- a/irods/src/test/java/ch/cyberduck/core/irods/IRODSFindFeatureTest.java +++ b/irods/src/test/java/ch/cyberduck/core/irods/IRODSFindFeatureTest.java @@ -29,8 +29,7 @@ import ch.cyberduck.core.proxy.DisabledProxyFinder; import ch.cyberduck.core.serializer.impl.dd.ProfilePlistReader; import ch.cyberduck.core.transfer.TransferStatus; -import ch.cyberduck.test.IntegrationTest; -import ch.cyberduck.test.VaultTest; +import ch.cyberduck.test.TestcontainerTest; import org.junit.Test; import org.junit.experimental.categories.Category; @@ -43,14 +42,14 @@ import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertTrue; -@Category(IntegrationTest.class) -public class IRODSFindFeatureTest extends VaultTest { +@Category(TestcontainerTest.class) +public class IRODSFindFeatureTest extends IRODSDockerComposeManager { @Test public void testFind() throws Exception { final ProtocolFactory factory = new ProtocolFactory(new HashSet<>(Collections.singleton(new IRODSProtocol()))); final Profile profile = new ProfilePlistReader(factory).read( - this.getClass().getResourceAsStream("/iRODS (iPlant Collaborative).cyberduckprofile")); + this.getClass().getResourceAsStream("/iRODS.cyberduckprofile")); final Host host = new Host(profile, profile.getDefaultHostname(), new Credentials( PROPERTIES.get("irods.key"), PROPERTIES.get("irods.secret") )); diff --git a/irods/src/test/java/ch/cyberduck/core/irods/IRODSListServiceTest.java b/irods/src/test/java/ch/cyberduck/core/irods/IRODSListServiceTest.java index 6b9b4cd806c..b0c0eff309c 100644 --- a/irods/src/test/java/ch/cyberduck/core/irods/IRODSListServiceTest.java +++ b/irods/src/test/java/ch/cyberduck/core/irods/IRODSListServiceTest.java @@ -29,12 +29,9 @@ import ch.cyberduck.core.ProtocolFactory; import ch.cyberduck.core.exception.NotfoundException; import ch.cyberduck.core.proxy.DisabledProxyFinder; -import ch.cyberduck.core.proxy.Proxy; import ch.cyberduck.core.serializer.impl.dd.ProfilePlistReader; -import ch.cyberduck.test.IntegrationTest; -import ch.cyberduck.test.VaultTest; +import ch.cyberduck.test.TestcontainerTest; -import org.junit.Ignore; import org.junit.Test; import org.junit.experimental.categories.Category; @@ -45,15 +42,14 @@ import static org.junit.Assert.*; -@Category(IntegrationTest.class) -public class IRODSListServiceTest extends VaultTest { +@Category(TestcontainerTest.class) +public class IRODSListServiceTest extends IRODSDockerComposeManager { @Test - @Ignore public void testList() throws Exception { final ProtocolFactory factory = new ProtocolFactory(new HashSet<>(Collections.singleton(new IRODSProtocol()))); final Profile profile = new ProfilePlistReader(factory).read( - this.getClass().getResourceAsStream("/iRODS (iPlant Collaborative).cyberduckprofile")); + this.getClass().getResourceAsStream("/iRODS.cyberduckprofile")); final Host host = new Host(profile, profile.getDefaultHostname(), new Credentials( PROPERTIES.get("irods.key"), PROPERTIES.get("irods.secret") )); @@ -65,7 +61,7 @@ public void testList() throws Exception { assertNotNull(session.getClient()); session.login(new DisabledLoginCallback(), new DisabledCancelCallback()); final AttributedList list = new IRODSListService(session).list(new IRODSHomeFinderService(session).find(), new DisabledListProgressListener()); - assertFalse(list.isEmpty()); + assertTrue(list.isEmpty()); for(Path p : list) { assertEquals(new IRODSHomeFinderService(session).find(), p.getParent()); assertNotEquals(-1L, p.attributes().getModificationDate()); @@ -78,7 +74,7 @@ public void testList() throws Exception { public void testListNotfound() throws Exception { final ProtocolFactory factory = new ProtocolFactory(new HashSet<>(Collections.singleton(new IRODSProtocol()))); final Profile profile = new ProfilePlistReader(factory).read( - this.getClass().getResourceAsStream("/iRODS (iPlant Collaborative).cyberduckprofile")); + this.getClass().getResourceAsStream("/iRODS.cyberduckprofile")); final Host host = new Host(profile, profile.getDefaultHostname(), new Credentials( PROPERTIES.get("irods.key"), PROPERTIES.get("irods.secret") )); diff --git a/irods/src/test/java/ch/cyberduck/core/irods/IRODSMoveFeatureTest.java b/irods/src/test/java/ch/cyberduck/core/irods/IRODSMoveFeatureTest.java index ca9a7a90d45..7a6dd1016cb 100644 --- a/irods/src/test/java/ch/cyberduck/core/irods/IRODSMoveFeatureTest.java +++ b/irods/src/test/java/ch/cyberduck/core/irods/IRODSMoveFeatureTest.java @@ -33,8 +33,7 @@ import ch.cyberduck.core.proxy.DisabledProxyFinder; import ch.cyberduck.core.serializer.impl.dd.ProfilePlistReader; import ch.cyberduck.core.transfer.TransferStatus; -import ch.cyberduck.test.IntegrationTest; -import ch.cyberduck.test.VaultTest; +import ch.cyberduck.test.TestcontainerTest; import org.junit.Test; import org.junit.experimental.categories.Category; @@ -46,14 +45,14 @@ import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertTrue; -@Category(IntegrationTest.class) -public class IRODSMoveFeatureTest extends VaultTest { +@Category(TestcontainerTest.class) +public class IRODSMoveFeatureTest extends IRODSDockerComposeManager { @Test public void testMoveDirectory() throws Exception { final ProtocolFactory factory = new ProtocolFactory(new HashSet<>(Collections.singleton(new IRODSProtocol()))); final Profile profile = new ProfilePlistReader(factory).read( - this.getClass().getResourceAsStream("/iRODS (iPlant Collaborative).cyberduckprofile")); + this.getClass().getResourceAsStream("/iRODS.cyberduckprofile")); final Host host = new Host(profile, profile.getDefaultHostname(), new Credentials( PROPERTIES.get("irods.key"), PROPERTIES.get("irods.secret") )); @@ -83,7 +82,7 @@ public void testMoveDirectory() throws Exception { public void testMoveFile() throws Exception { final ProtocolFactory factory = new ProtocolFactory(new HashSet<>(Collections.singleton(new IRODSProtocol()))); final Profile profile = new ProfilePlistReader(factory).read( - this.getClass().getResourceAsStream("/iRODS (iPlant Collaborative).cyberduckprofile")); + this.getClass().getResourceAsStream("/iRODS.cyberduckprofile")); final Host host = new Host(profile, profile.getDefaultHostname(), new Credentials( PROPERTIES.get("irods.key"), PROPERTIES.get("irods.secret") )); @@ -108,7 +107,7 @@ public void testMoveFile() throws Exception { public void testMoveNotFound() throws Exception { final ProtocolFactory factory = new ProtocolFactory(new HashSet<>(Collections.singleton(new IRODSProtocol()))); final Profile profile = new ProfilePlistReader(factory).read( - this.getClass().getResourceAsStream("/iRODS (iPlant Collaborative).cyberduckprofile")); + this.getClass().getResourceAsStream("/iRODS.cyberduckprofile")); final Host host = new Host(profile, profile.getDefaultHostname(), new Credentials( PROPERTIES.get("irods.key"), PROPERTIES.get("irods.secret") )); diff --git a/irods/src/test/java/ch/cyberduck/core/irods/IRODSReadFeatureTest.java b/irods/src/test/java/ch/cyberduck/core/irods/IRODSReadFeatureTest.java index e77c9083589..91f11b7267f 100644 --- a/irods/src/test/java/ch/cyberduck/core/irods/IRODSReadFeatureTest.java +++ b/irods/src/test/java/ch/cyberduck/core/irods/IRODSReadFeatureTest.java @@ -38,8 +38,7 @@ import ch.cyberduck.core.serializer.impl.dd.ProfilePlistReader; import ch.cyberduck.core.shared.DefaultUploadFeature; import ch.cyberduck.core.transfer.TransferStatus; -import ch.cyberduck.test.IntegrationTest; -import ch.cyberduck.test.VaultTest; +import ch.cyberduck.test.TestcontainerTest; import org.apache.commons.io.IOUtils; import org.apache.commons.lang3.RandomUtils; @@ -57,14 +56,14 @@ import static org.junit.Assert.*; -@Category(IntegrationTest.class) -public class IRODSReadFeatureTest extends VaultTest { +@Category(TestcontainerTest.class) +public class IRODSReadFeatureTest extends IRODSDockerComposeManager { @Test public void testRead() throws Exception { final ProtocolFactory factory = new ProtocolFactory(new HashSet<>(Collections.singleton(new IRODSProtocol()))); final Profile profile = new ProfilePlistReader(factory).read( - this.getClass().getResourceAsStream("/iRODS (iPlant Collaborative).cyberduckprofile")); + this.getClass().getResourceAsStream("/iRODS.cyberduckprofile")); final Host host = new Host(profile, profile.getDefaultHostname(), new Credentials( PROPERTIES.get("irods.key"), PROPERTIES.get("irods.secret") )); @@ -100,7 +99,7 @@ public void testRead() throws Exception { public void testReadNotFound() throws Exception { final ProtocolFactory factory = new ProtocolFactory(new HashSet<>(Collections.singleton(new IRODSProtocol()))); final Profile profile = new ProfilePlistReader(factory).read( - this.getClass().getResourceAsStream("/iRODS (iPlant Collaborative).cyberduckprofile")); + this.getClass().getResourceAsStream("/iRODS.cyberduckprofile")); final Host host = new Host(profile, profile.getDefaultHostname(), new Credentials( PROPERTIES.get("irods.key"), PROPERTIES.get("irods.secret") )); @@ -120,7 +119,7 @@ public void testReadNotFound() throws Exception { public void testReadRange() throws Exception { final ProtocolFactory factory = new ProtocolFactory(new HashSet<>(Collections.singleton(new IRODSProtocol()))); final Profile profile = new ProfilePlistReader(factory).read( - this.getClass().getResourceAsStream("/iRODS (iPlant Collaborative).cyberduckprofile")); + this.getClass().getResourceAsStream("/iRODS.cyberduckprofile")); final Host host = new Host(profile, profile.getDefaultHostname(), new Credentials( PROPERTIES.get("irods.key"), PROPERTIES.get("irods.secret") )); diff --git a/irods/src/test/java/ch/cyberduck/core/irods/IRODSSessionTest.java b/irods/src/test/java/ch/cyberduck/core/irods/IRODSSessionTest.java index 160b00ee40f..7db1304fa4a 100644 --- a/irods/src/test/java/ch/cyberduck/core/irods/IRODSSessionTest.java +++ b/irods/src/test/java/ch/cyberduck/core/irods/IRODSSessionTest.java @@ -29,10 +29,8 @@ import ch.cyberduck.core.ProtocolFactory; import ch.cyberduck.core.exception.LoginFailureException; import ch.cyberduck.core.proxy.DisabledProxyFinder; -import ch.cyberduck.core.proxy.Proxy; import ch.cyberduck.core.serializer.impl.dd.ProfilePlistReader; -import ch.cyberduck.test.IntegrationTest; -import ch.cyberduck.test.VaultTest; +import ch.cyberduck.test.TestcontainerTest; import org.junit.Test; import org.junit.experimental.categories.Category; @@ -42,14 +40,14 @@ import static org.junit.Assert.*; -@Category(IntegrationTest.class) -public class IRODSSessionTest extends VaultTest { +@Category(TestcontainerTest.class) +public class IRODSSessionTest extends IRODSDockerComposeManager { @Test public void testConnect() throws Exception { final ProtocolFactory factory = new ProtocolFactory(new HashSet<>(Collections.singleton(new IRODSProtocol()))); final Profile profile = new ProfilePlistReader(factory).read( - this.getClass().getResourceAsStream("/iRODS (iPlant Collaborative).cyberduckprofile")); + this.getClass().getResourceAsStream("/iRODS.cyberduckprofile")); final Host host = new Host(profile, profile.getDefaultHostname(), new Credentials( PROPERTIES.get("irods.key"), PROPERTIES.get("irods.secret") )); @@ -68,7 +66,7 @@ public void testConnect() throws Exception { public void testLoginDefault() throws Exception { final ProtocolFactory factory = new ProtocolFactory(new HashSet<>(Collections.singleton(new IRODSProtocol()))); final Profile profile = new ProfilePlistReader(factory).read( - this.getClass().getResourceAsStream("/iRODS (iPlant Collaborative).cyberduckprofile")); + this.getClass().getResourceAsStream("/iRODS.cyberduckprofile")); final Host host = new Host(profile, profile.getDefaultHostname(), new Credentials( PROPERTIES.get("irods.key"), PROPERTIES.get("irods.secret") )); @@ -82,7 +80,7 @@ public void testLoginDefault() throws Exception { session.login(new DisabledLoginCallback(), new DisabledCancelCallback()); final AttributedList list = new IRODSListService(session).list(new IRODSHomeFinderService(session).find(), new DisabledListProgressListener()); - assertFalse(list.isEmpty()); + assertTrue(list.isEmpty()); assertTrue(session.isConnected()); session.close(); @@ -93,7 +91,7 @@ public void testLoginDefault() throws Exception { public void testLoginWhitespaceHomeDirectory() throws Exception { final ProtocolFactory factory = new ProtocolFactory(new HashSet<>(Collections.singleton(new IRODSProtocol()))); final Profile profile = new ProfilePlistReader(factory).read( - this.getClass().getResourceAsStream("/iRODS (iPlant Collaborative).cyberduckprofile")); + this.getClass().getResourceAsStream("/iRODS.cyberduckprofile")); final Host host = new Host(profile, profile.getDefaultHostname(), new Credentials( PROPERTIES.get("irods.key"), PROPERTIES.get("irods.secret") )); @@ -116,7 +114,7 @@ public void testLoginWhitespaceHomeDirectory() throws Exception { public void testLoginFailure() throws Exception { final ProtocolFactory factory = new ProtocolFactory(new HashSet<>(Collections.singleton(new IRODSProtocol()))); final Profile profile = new ProfilePlistReader(factory).read( - this.getClass().getResourceAsStream("/iRODS (iPlant Collaborative).cyberduckprofile")); + this.getClass().getResourceAsStream("/iRODS.cyberduckprofile")); final Host host = new Host(profile, profile.getDefaultHostname(), new Credentials("a", "a")); final IRODSSession session = new IRODSSession(host); diff --git a/irods/src/test/java/ch/cyberduck/core/irods/IRODSTouchFeatureTest.java b/irods/src/test/java/ch/cyberduck/core/irods/IRODSTouchFeatureTest.java index a7a9fb03fc1..8b68b5f76ad 100644 --- a/irods/src/test/java/ch/cyberduck/core/irods/IRODSTouchFeatureTest.java +++ b/irods/src/test/java/ch/cyberduck/core/irods/IRODSTouchFeatureTest.java @@ -29,8 +29,7 @@ import ch.cyberduck.core.proxy.DisabledProxyFinder; import ch.cyberduck.core.serializer.impl.dd.ProfilePlistReader; import ch.cyberduck.core.transfer.TransferStatus; -import ch.cyberduck.test.IntegrationTest; -import ch.cyberduck.test.VaultTest; +import ch.cyberduck.test.TestcontainerTest; import org.junit.Test; import org.junit.experimental.categories.Category; @@ -43,14 +42,14 @@ import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertTrue; -@Category(IntegrationTest.class) -public class IRODSTouchFeatureTest extends VaultTest { +@Category(TestcontainerTest.class) +public class IRODSTouchFeatureTest extends IRODSDockerComposeManager { @Test public void testTouch() throws Exception { final ProtocolFactory factory = new ProtocolFactory(new HashSet<>(Collections.singleton(new IRODSProtocol()))); final Profile profile = new ProfilePlistReader(factory).read( - this.getClass().getResourceAsStream("/iRODS (iPlant Collaborative).cyberduckprofile")); + this.getClass().getResourceAsStream("/iRODS.cyberduckprofile")); final Host host = new Host(profile, profile.getDefaultHostname(), new Credentials( PROPERTIES.get("irods.key"), PROPERTIES.get("irods.secret") )); diff --git a/irods/src/test/java/ch/cyberduck/core/irods/IRODSUploadFeatureTest.java b/irods/src/test/java/ch/cyberduck/core/irods/IRODSUploadFeatureTest.java index 947a3ac28e5..b80402751e1 100644 --- a/irods/src/test/java/ch/cyberduck/core/irods/IRODSUploadFeatureTest.java +++ b/irods/src/test/java/ch/cyberduck/core/irods/IRODSUploadFeatureTest.java @@ -36,8 +36,7 @@ import ch.cyberduck.core.proxy.DisabledProxyFinder; import ch.cyberduck.core.serializer.impl.dd.ProfilePlistReader; import ch.cyberduck.core.transfer.TransferStatus; -import ch.cyberduck.test.IntegrationTest; -import ch.cyberduck.test.VaultTest; +import ch.cyberduck.test.TestcontainerTest; import org.apache.commons.io.IOUtils; import org.apache.commons.lang3.RandomUtils; @@ -54,15 +53,15 @@ import static org.junit.Assert.*; -@Category(IntegrationTest.class) -public class IRODSUploadFeatureTest extends VaultTest { +@Category(TestcontainerTest.class) +public class IRODSUploadFeatureTest extends IRODSDockerComposeManager { @Test @Ignore public void testAppend() throws Exception { final ProtocolFactory factory = new ProtocolFactory(new HashSet<>(Collections.singleton(new IRODSProtocol()))); final Profile profile = new ProfilePlistReader(factory).read( - this.getClass().getResourceAsStream("/iRODS (iPlant Collaborative).cyberduckprofile")); + this.getClass().getResourceAsStream("/iRODS.cyberduckprofile")); final Host host = new Host(profile, profile.getDefaultHostname(), new Credentials( PROPERTIES.get("irods.key"), PROPERTIES.get("irods.secret") )); @@ -108,7 +107,7 @@ public void testAppend() throws Exception { public void testWrite() throws Exception { final ProtocolFactory factory = new ProtocolFactory(new HashSet<>(Collections.singleton(new IRODSProtocol()))); final Profile profile = new ProfilePlistReader(factory).read( - this.getClass().getResourceAsStream("/iRODS (iPlant Collaborative).cyberduckprofile")); + this.getClass().getResourceAsStream("/iRODS.cyberduckprofile")); final Host host = new Host(profile, profile.getDefaultHostname(), new Credentials( PROPERTIES.get("irods.key"), PROPERTIES.get("irods.secret") )); @@ -139,10 +138,10 @@ public void testWrite() throws Exception { } @Test - public void testInterruptStatus() throws Exception { + public void testInterruptStatusForSmallFiles() throws Exception { final ProtocolFactory factory = new ProtocolFactory(new HashSet<>(Collections.singleton(new IRODSProtocol()))); final Profile profile = new ProfilePlistReader(factory).read( - this.getClass().getResourceAsStream("/iRODS (iPlant Collaborative).cyberduckprofile")); + this.getClass().getResourceAsStream("/iRODS.cyberduckprofile")); final Host host = new Host(profile, profile.getDefaultHostname(), new Credentials( PROPERTIES.get("irods.key"), PROPERTIES.get("irods.secret") )); @@ -159,7 +158,10 @@ public void testInterruptStatus() throws Exception { final Path test = new Path(new IRODSHomeFinderService(session).find(), UUID.randomUUID().toString(), EnumSet.of(Path.Type.file)); final TransferStatus status = new TransferStatus().setLength(content.length); new IRODSUploadFeature(session).upload( - new IRODSWriteFeature(session), test, local, new BandwidthThrottle(BandwidthThrottle.UNLIMITED), new DisabledProgressListener(), new DisabledStreamListener() { + new IRODSWriteFeature(session), test, local, + new BandwidthThrottle(BandwidthThrottle.UNLIMITED), + new DisabledProgressListener(), + new DisabledStreamListener() { @Override public void sent(final long bytes) { super.sent(bytes); @@ -176,6 +178,52 @@ public void sent(final long bytes) { // } assertFalse(status.isComplete()); + new IRODSDeleteFeature(session).delete(Collections.singletonList(test), new DisabledLoginCallback(), new Delete.DisabledCallback()); + session.close(); + } + + @Test + public void testInterruptStatusForLargeFiles() throws Exception { + final ProtocolFactory factory = new ProtocolFactory(new HashSet<>(Collections.singleton(new IRODSProtocol()))); + final Profile profile = new ProfilePlistReader(factory).read( + this.getClass().getResourceAsStream("/iRODS.cyberduckprofile")); + final Host host = new Host(profile, profile.getDefaultHostname(), new Credentials( + PROPERTIES.get("irods.key"), PROPERTIES.get("irods.secret") + )); + + final IRODSSession session = new IRODSSession(host); + session.open(new DisabledProxyFinder(), new DisabledHostKeyCallback(), new DisabledLoginCallback(), new DisabledCancelCallback()); + session.login(new DisabledLoginCallback(), new DisabledCancelCallback()); + final Local local = new Local(System.getProperty("java.io.tmpdir"), UUID.randomUUID().toString()); + final int length = 33 * 1024 * 1024; // Triggers parallel transfer + final byte[] content = RandomUtils.nextBytes(length); + final OutputStream out = local.getOutputStream(false); + IOUtils.write(content, out); + out.close(); + final Path test = new Path(new IRODSHomeFinderService(session).find(), UUID.randomUUID().toString(), EnumSet.of(Path.Type.file)); + final TransferStatus status = new TransferStatus().setLength(content.length); + new IRODSUploadFeature(session).upload( + new IRODSWriteFeature(session), test, local, + new BandwidthThrottle(BandwidthThrottle.UNLIMITED), + new DisabledProgressListener(), + new DisabledStreamListener() { + @Override + public void sent(final long bytes) { + super.sent(bytes); + status.setCanceled(); + } + }, + status, + new DisabledConnectionCallback()); + try { + status.validate(); + fail(); + } + catch(ConnectionCanceledException e) { + // + } + assertFalse(status.isComplete()); + new IRODSDeleteFeature(session).delete(Collections.singletonList(test), new DisabledLoginCallback(), new Delete.DisabledCallback()); session.close(); } } diff --git a/irods/src/test/java/ch/cyberduck/core/irods/IRODSWriteFeatureTest.java b/irods/src/test/java/ch/cyberduck/core/irods/IRODSWriteFeatureTest.java index 4212f1f739d..1a71866c0e6 100644 --- a/irods/src/test/java/ch/cyberduck/core/irods/IRODSWriteFeatureTest.java +++ b/irods/src/test/java/ch/cyberduck/core/irods/IRODSWriteFeatureTest.java @@ -36,8 +36,7 @@ import ch.cyberduck.core.proxy.DisabledProxyFinder; import ch.cyberduck.core.serializer.impl.dd.ProfilePlistReader; import ch.cyberduck.core.transfer.TransferStatus; -import ch.cyberduck.test.IntegrationTest; -import ch.cyberduck.test.VaultTest; +import ch.cyberduck.test.TestcontainerTest; import org.apache.commons.io.IOUtils; import org.apache.commons.lang3.RandomUtils; @@ -55,14 +54,14 @@ import static org.junit.Assert.*; -@Category(IntegrationTest.class) -public class IRODSWriteFeatureTest extends VaultTest { +@Category(TestcontainerTest.class) +public class IRODSWriteFeatureTest extends IRODSDockerComposeManager { @Test public void testWriteConcurrent() throws Exception { final ProtocolFactory factory = new ProtocolFactory(new HashSet<>(Collections.singleton(new IRODSProtocol()))); final Profile profile = new ProfilePlistReader(factory).read( - this.getClass().getResourceAsStream("/iRODS (iPlant Collaborative).cyberduckprofile")); + this.getClass().getResourceAsStream("/iRODS.cyberduckprofile")); final Host host = new Host(profile, profile.getDefaultHostname(), new Credentials( PROPERTIES.get("irods.key"), PROPERTIES.get("irods.secret") )); @@ -100,7 +99,13 @@ public void testWriteConcurrent() throws Exception { in2.close(); assertArrayEquals(content, buffer2); } + + session1.getFeature(Delete.class).delete(Collections.singletonList(test1), new DisabledLoginCallback(), new Delete.DisabledCallback()); + assertFalse(session1.getFeature(Find.class).find(test1)); session1.close(); + + session2.getFeature(Delete.class).delete(Collections.singletonList(test2), new DisabledLoginCallback(), new Delete.DisabledCallback()); + assertFalse(session2.getFeature(Find.class).find(test2)); session2.close(); } @@ -108,7 +113,7 @@ public void testWriteConcurrent() throws Exception { public void testWriteThreaded() throws Exception { final ProtocolFactory factory = new ProtocolFactory(new HashSet<>(Collections.singleton(new IRODSProtocol()))); final Profile profile = new ProfilePlistReader(factory).read( - this.getClass().getResourceAsStream("/iRODS (iPlant Collaborative).cyberduckprofile")); + this.getClass().getResourceAsStream("/iRODS.cyberduckprofile")); final Host host = new Host(profile, profile.getDefaultHostname(), new Credentials( PROPERTIES.get("irods.key"), PROPERTIES.get("irods.secret") )); @@ -203,7 +208,12 @@ public void run() { cr1.await(); cr2.await(); + session1.getFeature(Delete.class).delete(Collections.singletonList(test1), new DisabledLoginCallback(), new Delete.DisabledCallback()); + assertFalse(session1.getFeature(Find.class).find(test1)); session1.close(); + + session2.getFeature(Delete.class).delete(Collections.singletonList(test2), new DisabledLoginCallback(), new Delete.DisabledCallback()); + assertFalse(session2.getFeature(Find.class).find(test2)); session2.close(); } @@ -211,7 +221,7 @@ public void run() { public void testWrite() throws Exception { final ProtocolFactory factory = new ProtocolFactory(new HashSet<>(Collections.singleton(new IRODSProtocol()))); final Profile profile = new ProfilePlistReader(factory).read( - this.getClass().getResourceAsStream("/iRODS (iPlant Collaborative).cyberduckprofile")); + this.getClass().getResourceAsStream("/iRODS.cyberduckprofile")); final Host host = new Host(profile, profile.getDefaultHostname(), new Credentials( PROPERTIES.get("irods.key"), PROPERTIES.get("irods.secret") )); @@ -260,7 +270,7 @@ public void testWrite() throws Exception { assertEquals(content.length, new IRODSUploadFeature(session).append(test, status).offset, 0L); final StatusOutputStream out = feature.write(test, status, new DisabledConnectionCallback()); - assertNull(out); + assertNotNull(out); new StreamCopier(new TransferStatus(), new TransferStatus()).transfer(new ByteArrayInputStream(newcontent), out); assertTrue(session.getFeature(Find.class).find(test)); @@ -284,7 +294,7 @@ public void testWrite() throws Exception { public void testWriteAppend() throws Exception { final ProtocolFactory factory = new ProtocolFactory(new HashSet<>(Collections.singleton(new IRODSProtocol()))); final Profile profile = new ProfilePlistReader(factory).read( - this.getClass().getResourceAsStream("/iRODS (iPlant Collaborative).cyberduckprofile")); + this.getClass().getResourceAsStream("/iRODS.cyberduckprofile")); final Host host = new Host(profile, profile.getDefaultHostname(), new Credentials( PROPERTIES.get("irods.key"), PROPERTIES.get("irods.secret") )); diff --git a/irods/src/test/resources/docker/docker-compose.yml b/irods/src/test/resources/docker/docker-compose.yml new file mode 100644 index 00000000000..9397c839929 --- /dev/null +++ b/irods/src/test/resources/docker/docker-compose.yml @@ -0,0 +1,27 @@ +name: cyberduck-irods-testing + +services: + irods-catalog: + build: + context: irods_catalog + environment: + - POSTGRES_PASSWORD=testpassword + restart: always + + irods-catalog-provider: + build: + context: irods_catalog_provider + init: true + ports: + - "1247:1247" + shm_size: 100mb + healthcheck: + test: ["CMD", 'echo -e "\x00\x00\x00\x33HEARTBEAT" | (exec 3<>/dev/tcp/127.0.0.1/1247; cat >&3; cat <&3; exec 3<&-)'] + interval: 10s + timeout: 10s + retries: 3 + start_period: 20s + start_interval: 10s + restart: always + depends_on: + - irods-catalog \ No newline at end of file diff --git a/irods/src/test/resources/docker/irods_catalog/Dockerfile b/irods/src/test/resources/docker/irods_catalog/Dockerfile new file mode 100644 index 00000000000..cfdacba890a --- /dev/null +++ b/irods/src/test/resources/docker/irods_catalog/Dockerfile @@ -0,0 +1,3 @@ +FROM postgres:17 + +COPY init-user-db.sh /docker-entrypoint-initdb.d/init-user-db.sh \ No newline at end of file diff --git a/irods/src/test/resources/docker/irods_catalog/init-user-db.sh b/irods/src/test/resources/docker/irods_catalog/init-user-db.sh new file mode 100644 index 00000000000..692340c5f9a --- /dev/null +++ b/irods/src/test/resources/docker/irods_catalog/init-user-db.sh @@ -0,0 +1,12 @@ +#! /bin/bash + +# Adapted from "Initialization script" in documentation for official Postgres dockerhub: +# https://hub.docker.com/_/postgres/ +set -e + +psql -v ON_ERROR_STOP=1 --username "$POSTGRES_USER" --dbname "$POSTGRES_DB" <<-EOSQL + CREATE USER irods WITH PASSWORD 'testpassword'; + CREATE DATABASE "ICAT"; + GRANT ALL PRIVILEGES ON DATABASE "ICAT" to irods; + ALTER DATABASE "ICAT" OWNER TO irods; +EOSQL \ No newline at end of file diff --git a/irods/src/test/resources/docker/irods_catalog_provider/Dockerfile b/irods/src/test/resources/docker/irods_catalog_provider/Dockerfile new file mode 100644 index 00000000000..8836f0fea61 --- /dev/null +++ b/irods/src/test/resources/docker/irods_catalog_provider/Dockerfile @@ -0,0 +1,61 @@ +FROM ubuntu:24.04 + +ENV DEBIAN_FRONTEND=noninteractive + +RUN apt-get update && \ + apt-get install -y \ + apt-transport-https \ + gnupg \ + wget \ + netcat-traditional \ + && \ + apt-get clean && \ + rm -rf /var/lib/apt/lists/* /tmp/* + +RUN mkdir -p /etc/apt/keyrings && \ + wget -qO - https://packages.irods.org/irods-signing-key.asc | \ + gpg \ + --no-options \ + --no-default-keyring \ + --no-auto-check-trustdb \ + --homedir /dev/null \ + --no-keyring \ + --import-options import-export \ + --output /etc/apt/keyrings/renci-irods-archive-keyring.pgp \ + --import \ + && \ + echo "deb [signed-by=/etc/apt/keyrings/renci-irods-archive-keyring.pgp arch=amd64] https://packages.irods.org/apt/ noble main" | \ + tee /etc/apt/sources.list.d/renci-irods.list + +RUN apt-get update && \ + apt-get install -y \ + libcurl4-gnutls-dev \ + python3 \ + python3-distro \ + python3-jsonschema \ + python3-pip \ + python3-psutil \ + python3-requests \ + rsyslog \ + unixodbc \ + && \ + apt-get clean && \ + rm -rf /var/lib/apt/lists/* /tmp/* + +ARG irods_version=5.0.2 +ARG irods_package_version_suffix=-0~noble +ARG irods_package_version=${irods_version}${irods_package_version_suffix} + +RUN apt-get update && \ + apt-get install -y \ + irods-database-plugin-postgres=${irods_package_version} \ + irods-runtime=${irods_package_version} \ + irods-server=${irods_package_version} \ + irods-icommands=${irods_package_version} \ + && \ + apt-get clean && \ + rm -rf /var/lib/apt/lists/* /tmp/* + +COPY unattended_install.json / +COPY --chmod=755 entrypoint.sh / +ENTRYPOINT ["./entrypoint.sh"] \ No newline at end of file diff --git a/irods/src/test/resources/docker/irods_catalog_provider/entrypoint.sh b/irods/src/test/resources/docker/irods_catalog_provider/entrypoint.sh new file mode 100644 index 00000000000..306cb564c15 --- /dev/null +++ b/irods/src/test/resources/docker/irods_catalog_provider/entrypoint.sh @@ -0,0 +1,24 @@ +#! /bin/bash -e + +echo "Waiting for iRODS catalog database to be ready" +catalog_db_hostname=irods-catalog +until pg_isready -h ${catalog_db_hostname} -d ICAT -U irods -q; do + sleep 1 +done +echo "iRODS catalog database is ready" + +unattended_install_file=/unattended_install.json +if [ -f "${unattended_install_file}" ]; then + echo "Running iRODS setup" + + # Add generated hostname as a recognizable alias. + sed -i "s/CONTAINER_HOSTNAME_ALIAS/${HOSTNAME}/g" ${unattended_install_file} + python3 /var/lib/irods/scripts/setup_irods.py --json_configuration_file ${unattended_install_file} + + # Move the input file used to configure the server out of the way so + # the container is restartable. + mv ${unattended_install_file} ${unattended_install_file}.processed +fi + +echo "Starting server" +su - irods -c 'irodsServer --stdout' diff --git a/irods/src/test/resources/docker/irods_catalog_provider/unattended_install.json b/irods/src/test/resources/docker/irods_catalog_provider/unattended_install.json new file mode 100644 index 00000000000..2369cea5d34 --- /dev/null +++ b/irods/src/test/resources/docker/irods_catalog_provider/unattended_install.json @@ -0,0 +1,169 @@ +{ + "admin_password": "rods", + "default_resource_directory": "/var/lib/irods/Vault", + "default_resource_name": "demoResc", + "host_system_information": { + "service_account_user_name": "irods", + "service_account_group_name": "irods" + }, + "service_account_environment": { + "irods_client_server_policy": "CS_NEG_REFUSE", + "irods_connection_pool_refresh_time_in_seconds": 300, + "irods_cwd": "/tempZone/home/rods", + "irods_default_hash_scheme": "SHA256", + "irods_default_number_of_transfer_threads": 4, + "irods_default_resource": "demoResc", + "irods_encryption_algorithm": "AES-256-CBC", + "irods_encryption_key_size": 32, + "irods_encryption_num_hash_rounds": 16, + "irods_encryption_salt_size": 8, + "irods_home": "/tempZone/home/rods", + "irods_host": "irods-catalog-provider", + "irods_match_hash_policy": "compatible", + "irods_maximum_size_for_single_buffer_in_megabytes": 32, + "irods_port": 1247, + "irods_transfer_buffer_size_for_parallel_transfer_in_megabytes": 4, + "irods_user_name": "rods", + "irods_zone_name": "tempZone", + "schema_name": "service_account_environment", + "schema_version": "v5" + }, + "server_config": { + "advanced_settings": { + "checksum_read_buffer_size_in_bytes": 1048576, + "default_number_of_transfer_threads": 4, + "default_temporary_password_lifetime_in_seconds": 120, + "delay_rule_executors": [], + "delay_server_sleep_time_in_seconds": 30, + "dns_cache": { + "eviction_age_in_seconds": 3600, + "cache_clearer_sleep_time_in_seconds": 600, + "shared_memory_size_in_bytes": 5000000 + }, + "hostname_cache": { + "eviction_age_in_seconds": 3600, + "cache_clearer_sleep_time_in_seconds": 600, + "shared_memory_size_in_bytes": 2500000 + }, + "maximum_size_for_single_buffer_in_megabytes": 32, + "maximum_size_of_delay_queue_in_bytes": 0, + "maximum_temporary_password_lifetime_in_seconds": 1000, + "migrate_delay_server_sleep_time_in_seconds": 5, + "number_of_concurrent_delay_rule_executors": 4, + "stacktrace_file_processor_sleep_time_in_seconds": 10, + "transfer_buffer_size_for_parallel_transfer_in_megabytes": 4, + "transfer_chunk_size_for_parallel_transfer_in_megabytes": 40 + }, + "catalog_provider_hosts": [ + "irods-catalog-provider" + ], + "catalog_service_role": "provider", + "client_server_policy": "CS_NEG_REFUSE", + "connection_pool_refresh_time_in_seconds": 300, + "controlled_user_connection_list": { + "control_type": "denylist", + "users": [] + }, + "default_dir_mode": "0750", + "default_file_mode": "0600", + "default_hash_scheme": "SHA256", + "default_resource_name": "demoResc", + "encryption": { + "algorithm": "AES-256-CBC", + "key_size": 32, + "num_hash_rounds": 16, + "salt_size": 8 + }, + "environment_variables": {}, + "federation": [], + "graceful_shutdown_timeout_in_seconds": 30, + "host": "irods-catalog-provider", + "host_access_control": { + "access_entries": [] + }, + "host_resolution": { + "host_entries": [ + { + "address_type": "local", + "addresses": [ + "irods-catalog-provider", + "CONTAINER_HOSTNAME_ALIAS" + ] + } + ] + }, + "log_level": { + "agent": "info", + "agent_factory": "info", + "api": "info", + "authentication": "info", + "database": "info", + "delay_server": "info", + "genquery1": "info", + "genquery2": "info", + "legacy": "info", + "microservice": "info", + "network": "info", + "resource": "info", + "rule_engine": "info", + "server": "info", + "sql": "info" + }, + "match_hash_policy": "compatible", + "negotiation_key": "32_byte_server_negotiation_key__", + "plugin_configuration": { + "authentication": {}, + "database": { + "technology": "postgres", + "host": "irods-catalog", + "name": "ICAT", + "odbc_driver": "PostgreSQL ANSI", + "password": "testpassword", + "port": 5432, + "username": "irods" + }, + "network": {}, + "resource": {}, + "rule_engines": [ + { + "instance_name": "irods_rule_engine_plugin-irods_rule_language-instance", + "plugin_name": "irods_rule_engine_plugin-irods_rule_language", + "plugin_specific_configuration": { + "re_data_variable_mapping_set": [ + "core" + ], + "re_function_name_mapping_set": [ + "core" + ], + "re_rulebase_set": [ + "core" + ], + "regexes_for_supported_peps": [ + "ac[^ ]*", + "msi[^ ]*", + "[^ ]*pep_[^ ]*_(pre|post|except|finally)" + ] + }, + "shared_memory_instance": "irods_rule_language_rule_engine" + }, + { + "instance_name": "irods_rule_engine_plugin-cpp_default_policy-instance", + "plugin_name": "irods_rule_engine_plugin-cpp_default_policy", + "plugin_specific_configuration": {} + } + ] + }, + "rule_engine_namespaces": [ + "" + ], + "schema_name": "server_config", + "schema_version": "v5", + "server_port_range_end": 20199, + "server_port_range_start": 20000, + "zone_auth_scheme": "native", + "zone_key": "TEMPORARY_ZONE_KEY", + "zone_name": "tempZone", + "zone_port": 1247, + "zone_user": "rods" + } +} \ No newline at end of file diff --git a/irods/src/test/resources/iRODS (iPlant Collaborative).cyberduckprofile b/irods/src/test/resources/iRODS.cyberduckprofile similarity index 59% rename from irods/src/test/resources/iRODS (iPlant Collaborative).cyberduckprofile rename to irods/src/test/resources/iRODS.cyberduckprofile index 7d5484f947f..ab84678a88f 100644 --- a/irods/src/test/resources/iRODS (iPlant Collaborative).cyberduckprofile +++ b/irods/src/test/resources/iRODS.cyberduckprofile @@ -5,24 +5,35 @@ Protocol irods Vendor - iplant + iRODS Consortium Description - iPlant Data Store + iRODS (Integrated Rule-Oriented Data System) + Hostname Configurable Port Configurable + Default Hostname - data.iplantcollaborative.org - Region - iplant + localhost Default Port 1247 + Username Placeholder - iPlant username + iRODS username Password Placeholder - iPlant password + iRODS password + + Region + tempZone + Authorization - STANDARD + native + + Properties + + Delete Objects Permanently + yes +