diff --git a/pom.xml b/pom.xml index aa9b0b7f..e817f971 100644 --- a/pom.xml +++ b/pom.xml @@ -87,51 +87,6 @@ - - io.minio - minio - 5.0.2 - - - commons-codec - commons-codec - - - commons-logging - commons-logging - - - com.google.code.findbugs - annotations - - - com.google.code.findbugs - jsr305 - - - com.fasterxml.jackson.core - jackson-databind - - - com.fasterxml.jackson.core - jackson-core - - - com.fasterxml.jackson.core - jackson-annotations - - - - - com.fasterxml.jackson.core - jackson-databind - 2.14.2 - - - com.fasterxml.jackson.core - jackson-annotations - 2.14.2 - com.esotericsoftware kryo diff --git a/src/main/java/loci/common/Location.java b/src/main/java/loci/common/Location.java index 3d552dee..df5e078b 100644 --- a/src/main/java/loci/common/Location.java +++ b/src/main/java/loci/common/Location.java @@ -203,14 +203,8 @@ public Location(String parent, String child) { pathname = child; uri = new URI(mapped); isURL = true; - if (S3Handle.canHandleScheme(uri.toString())) { - urlType = UrlType.S3; - url = null; - } - else { - urlType = UrlType.GENERIC; - url = uri.toURL(); - } + urlType = UrlType.GENERIC; + url = uri.toURL(); } catch (URISyntaxException | MalformedURLException e) { // Readers such as FilePatternReader may pass invalid URI paths @@ -485,23 +479,8 @@ public static IRandomAccess getHandle(String id, boolean writable, LOGGER.trace("no handle was mapped for this ID"); String mapId = getMappedId(id); - if (S3Handle.canHandleScheme(id)) { - StreamHandle.Settings ss = new StreamHandle.Settings(); - if (ss.getRemoteCacheRootDir() != null) { - String cachedFile = S3Handle.cacheObject(mapId, ss); - if (bufferSize > 0) { - handle = new NIOFileHandle( - new File(cachedFile), "r", bufferSize); - } - else { - handle = new NIOFileHandle(cachedFile, "r"); - } - } - else { - handle = new S3Handle(mapId); - } - } - else if (id.startsWith("http://") || id.startsWith("https://")) { + if (id.startsWith("http://") || id.startsWith("https://") + || id.startsWith("s3://")) { handle = new URLHandle(mapId); } else if (allowArchiveHandles && ZipHandle.isZipFile(mapId)) { @@ -917,32 +896,9 @@ public boolean isAbsolute() { public boolean isDirectory() { LOGGER.trace("isDirectory()"); if (isURL) { - if (urlType == UrlType.S3) { - // TODO: This is complicated - // - // S3 doesn't have directories, but keys can contain / which we - // can pretend is a file path. However this "directory" doesn't - // actually exist, only the "contents" of the directory exist. - // - // Minio.listObjects() lists all objects in a bucket that - // match an optional prefix so this could be an option for checking - // whether to trest this as a directory. - // - // S3 buckets are the closest thing to a proper directory - // so for now - try { - S3Handle h = new S3Handle(uri.toString()); - boolean isBucket = h.isBucket(); - h.close(); - return isBucket; - } catch (IOException e) { - throw new UncheckedIOException(e); - } - } else { - // TODO: this should be removed as well. - String[] list = list(); - return list != null; - } + // TODO: this should be removed as well. + String[] list = list(); + return list != null; } return file.isDirectory(); } diff --git a/src/main/java/loci/common/S3Handle.java b/src/main/java/loci/common/S3Handle.java deleted file mode 100644 index 6a323c69..00000000 --- a/src/main/java/loci/common/S3Handle.java +++ /dev/null @@ -1,442 +0,0 @@ -/* - * #%L - * Common package for I/O and related utilities - * %% - * Copyright (C) 2018 Open Microscopy Environment: - * - Board of Regents of the University of Wisconsin-Madison - * - Glencoe Software, Inc. - * - University of Dundee - * %% - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions are met: - * - * 1. Redistributions of source code must retain the above copyright notice, - * this list of conditions and the following disclaimer. - * 2. Redistributions in binary form must reproduce the above copyright notice, - * this list of conditions and the following disclaimer in the documentation - * and/or other materials provided with the distribution. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" - * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE - * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE - * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDERS OR CONTRIBUTORS BE - * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR - * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF - * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS - * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN - * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) - * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE - * POSSIBILITY OF SUCH DAMAGE. - * #L% - */ - -package loci.common; - -import java.io.BufferedInputStream; -import java.io.DataInputStream; -import java.io.IOException; -import java.net.URI; -import java.net.URISyntaxException; -import java.nio.file.Files; -import java.nio.file.Path; -import java.nio.file.Paths; -import java.util.regex.Pattern; - -import loci.common.services.DependencyException; -import loci.common.services.S3ClientService; -import loci.common.services.S3ClientServiceException; -import loci.common.services.S3ClientStat; - -import loci.common.services.ServiceFactory; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -/** - * Provides random access to S3 buckets using the IRandomAccess interface. - * Instances of S3Handle are read-only. - * - * @see IRandomAccess - * @see StreamHandle - * @see java.net.URLConnection - * - */ -public class S3Handle extends StreamHandle { - - /** - * An S3 IOException that was not thrown immediately - */ - class DelayedObjectNotFound extends IOException { - DelayedObjectNotFound(S3Handle s3) { - super(String.format("Object not found: [%s] %s", s3, s3.objectNotFound), s3.objectNotFound); - } - } - - /** Default protocol for fetching s3:// */ - public final static String DEFAULT_S3_PROTOCOL = "https"; - - private static final Logger LOGGER = LoggerFactory.getLogger(S3Handle.class); - - protected final static Pattern SCHEME_PARSER = Pattern.compile("s3(\\+\\p{Alnum}+)?(://.*)?"); - - /** S3 configuration */ - private final Settings settings; - - /** Parsed URI used to configure this handle */ - private final URI uri; - - /** access key, if provided */ - private final String accessKey; - - /** secret key, if provided */ - private final String secretKey; - - /** name of the bucket */ - private final String bucket; - - /** endpoint to which requests will be sent */ - private final String server; - - /** port at the given server */ - private final int port; - - /** remaining path, or key, for this accessed resource */ - private final String path; - - /** S3 client */ - private S3ClientService s3Client; - - /** Remote file stat */ - private S3ClientStat stat; - - /** Is this a directory (currently only buckets are considered directories */ - private boolean isBucket; - - /** - * Exception if thrown during construction - */ - private Throwable objectNotFound; - - /** If seeking more than this distance reset and reopen at offset */ - protected static final int S3_MAX_FORWARD_SEEK = 1048576; - - /** - * Return true if this is a URL with an s3 scheme - * @param url URL - * @return true if this class can handle url - */ - public static boolean canHandleScheme(String url) { - return SCHEME_PARSER.matcher(url).matches(); - } - - /** - * Open an S3 file - * - * @param url the full URL to the S3 resource - * @throws IOException if there is an error during opening - */ - public S3Handle(String url) throws IOException { - this(url, true, null); - } - - /** - * Open an S3 file - * - * @param uristr the full URL to the S3 resource - * @param initialize If true open the stream, otherwise just parse connection - * string - * @param s custom settings object - * @throws IOException if there is an error during opening - */ - public S3Handle(String uristr, boolean initialize, Settings s) throws - IOException { - if (s == null) { - this.settings = new StreamHandle.Settings(); - } - else { - this.settings = s; - } - - try { - this.uri = new URI(uristr); - } catch (URISyntaxException e) { - throw new RuntimeException("Invalid URI " + uristr, e); - } - - // access[:secret] - String auth = this.uri.getUserInfo(); - String accessKey = null; - String secretKey = null; - if (auth != null) { - String[] authparts = auth.split(":", 2); - accessKey = authparts[0]; - if (authparts.length > 1) { - secretKey = authparts[1]; - } - } - this.accessKey = accessKey; - this.secretKey = secretKey; - - String protocol; - String scheme = this.uri.getScheme(); - if (scheme.equals("s3")) { - protocol = DEFAULT_S3_PROTOCOL; - } - else if (scheme.startsWith("s3+")) { - protocol = scheme.substring(3); - } - else { - protocol = scheme; - } - this.server = protocol + "://" + this.uri.getHost(); - - if (this.uri.getPort() == -1) { - this.port = 0; - } - else { - this.port = this.uri.getPort(); - } - - // First path component is the bucket - // TODO: Parsing this seems way more complicated than it should be - String fullpath = this.uri.getPath(); - if (fullpath == null || fullpath.length() == 0) { - fullpath = "/"; - } - // Leading / means first element is always "" - String[] pathparts = fullpath.split("/", 3); - if (pathparts[1].length() > 0) { - this.bucket = pathparts[1]; - } - else { - this.bucket = null; - } - if (pathparts.length > 2 && pathparts[2].length() > 0) { - this.path = pathparts[2]; - } - else { - this.path = null; - } - - this.isBucket = false; - this.stat = null; - - if (initialize) { - // Throw if there is an IOException, otherwise save the exception and only throw if a method - // that requires a valid object is called - this.connect(); - try { - this.initialize(); - } - catch (S3ClientServiceException e) { - this.objectNotFound = e; - LOGGER.debug("Object not found: [{}] {}", this, e); - } - LOGGER.trace("isBucket:{} stat:{}", isBucket, stat); - } - } - - /** - * Connect to the server - * @throws IOException if there was an error connecting to the server - */ - protected void connect() throws IOException { - final String appName = "Bio-Formats"; - // TODO: Replace "dev" with a version - final String appVersion = "dev"; - try { - ServiceFactory factory = new ServiceFactory(); - s3Client = factory.getInstance(S3ClientService.class); - s3Client.initialize(server, port, accessKey, secretKey, appName, appVersion); - } - catch (S3ClientServiceException e) { - throw new IOException(String.format( - "Failed to connect: %s", this), e); - } - catch (DependencyException e) { - throw new IOException(String.format( - "S3 requires additional dependencies: %s", this), e); - } - LOGGER.trace("connected: server:{} port:{}", server, port); - } - - /** - * Check bucket or object exists - * @throws IOException if unable to get the object - * @throws S3ClientServiceException if unable to get the object - */ - protected void initialize() throws - IOException, - S3ClientServiceException { - if (path == null) { - isBucket = s3Client.bucketExists(bucket); - } - else { - isBucket = false; - stat = s3Client.statObject(bucket, path); - resetStream(); - } - } - - public String getServer() { - return server; - } - - public int getPort() { - return port; - } - - public String getBucket() { - return bucket; - } - - public String getPath() { - return path; - } - - /** - * Download an S3 object to a file system cache if it doesn't already exist - * - * @param url the full URL to the S3 resource - * @param s custom settings object - * @return File path to the cached object - * @throws IOException if there is an error during reading or writing - * @throws HandleException if no destination for the cache is provided - */ - public static String cacheObject(String url, Settings s) throws - IOException, - HandleException { - String cacheroot = s.getRemoteCacheRootDir(); - if (cacheroot == null) { - throw new HandleException("Remote cache root dir is not set"); - } - S3Handle s3 = new S3Handle(url, true, s); - // TODO: Need to ensure this path is safe. Is there a Java method to check? - String cacheobj = s3.getCacheKey(); - // Hopefully creates a cross-platform path - Path cachepath = Paths.get(cacheroot, cacheobj); - - if (Files.exists(cachepath)) { - LOGGER.debug("Found existing cache for {} at {}", s3, cachepath); - } - else { - LOGGER.debug("Caching {} to {}", s3, cachepath); - s3.downloadObject(cachepath); - LOGGER.debug("Downloaded {}", cachepath); - } - return cachepath.toString(); - } - - public String getCacheKey(){ - String cachekey = - getServer().replace("://", "/") + "/" + - getPort() + "/" + - getBucket() + "/" + - getPath(); - return cachekey; - } - - protected void downloadObject(Path destination) throws HandleException, IOException { - LOGGER.trace("destination:{}", destination); - if (this.stat == null || this.objectNotFound != null) { - throw new IOException("Object not found " + this, this.objectNotFound); - } - if (path == null) { - throw new HandleException("Download path=null not allowed"); - } - Files.createDirectories(destination.getParent()); - try { - s3Client.getObject(bucket, path, destination.toString()); - } - catch (S3ClientServiceException e) { - throw new HandleException("Download failed " + toString(), e); - } - } - - /** - * Is this an accessible bucket? - * TODO: If this bucket doesn't exist do we return false or thrown an exception? - * - * @return True if a bucket - */ - public boolean isBucket() { - //if (this.objectNotFound != null) { - // throw new DelayedObjectNotFound(this); - //} - return isBucket; - } - - /* @see IRandomAccess#length() */ - @Override - public long length() throws IOException { - if (this.stat == null || this.objectNotFound != null) { - throw new DelayedObjectNotFound(this); - } - return length; - } - - /** - * @see StreamHandle#seek(long) - */ - @Override - public void seek(long pos) throws IOException { - LOGGER.trace("{}", pos); - if (this.stat == null || this.objectNotFound != null) { - throw new DelayedObjectNotFound(this); - } - long diff = pos - fp; - - if (diff < 0 || diff > S3_MAX_FORWARD_SEEK) { - resetStream(pos); - } - else { - super.seek(pos); - } - } - - /** - * @see StreamHandle#resetStream() - */ - @Override - protected void resetStream() throws IOException { - resetStream(0); - } - - /** - * Does this represent an accessible location? - * @return true if this location is accessible - * @throws IOException if unable to determine whether this location is accessible - */ - @Override - public boolean exists() throws IOException { - return (objectNotFound == null) && (isBucket || stat != null); - } - - /** - * Reset the stream to an offset position - * @param offset Offset into object - * @throws IOException if there is an error during reading or writing - */ - protected void resetStream(long offset) throws IOException { - LOGGER.trace("Resetting {}", offset); - if (this.stat == null || this.objectNotFound != null) { - throw new DelayedObjectNotFound(this); - } - try { - length = stat.length(); - stream = new DataInputStream(new BufferedInputStream( - s3Client.getObject(bucket, path, offset))); - fp = offset; - mark = offset; - } - catch (S3ClientServiceException e) { - throw new IOException(String.format( - "failed to load s3: %s\n\t%s", uri, this), e); - } - } - - public String toString() { - boolean found = (objectNotFound == null) && (isBucket || stat != null); - return String.format("server:%s port:%d bucket:%s path:%s found:%s", - server, port, bucket, path, found); - } -} diff --git a/src/main/java/loci/common/services/S3ClientService.java b/src/main/java/loci/common/services/S3ClientService.java deleted file mode 100644 index 55e046e0..00000000 --- a/src/main/java/loci/common/services/S3ClientService.java +++ /dev/null @@ -1,97 +0,0 @@ -/* - * #%L - * S3 client service - * %% - * Copyright (C) 2019 Open Microscopy Environment: - * - Board of Regents of the University of Wisconsin-Madison - * - Glencoe Software, Inc. - * - University of Dundee - * %% - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions are met: - * - * 1. Redistributions of source code must retain the above copyright notice, - * this list of conditions and the following disclaimer. - * 2. Redistributions in binary form must reproduce the above copyright notice, - * this list of conditions and the following disclaimer in the documentation - * and/or other materials provided with the distribution. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" - * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE - * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE - * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDERS OR CONTRIBUTORS BE - * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR - * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF - * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS - * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN - * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) - * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE - * POSSIBILITY OF SUCH DAMAGE. - * #L% - */ - -package loci.common.services; - -import java.io.IOException; -import java.io.InputStream; - -/** - * An S3 client - */ -public interface S3ClientService extends Service { - - /** - * Initialise the S3 client - * @param server servername - * @param port port - * @param accessKey access key - * @param secretKey secret key - * @param appName user agent application name - * @param appVersion user agent application version - * @throws S3ClientServiceException if an S3 error occurred - */ - void initialize(String server, int port, String accessKey, String secretKey, - String appName, String appVersion) - throws S3ClientServiceException; - - /** - * Check whether a bucket exists - * @param bucket Bucket name - * @return true if bucket exists - * @throws S3ClientServiceException if an S3 error occurred - * @throws IOException if an S3 error occurred - */ - boolean bucketExists(String bucket) throws S3ClientServiceException, IOException; - - /** - * Stat the object - * @param bucket Bucket name - * @param object Object path - * @return S3ClientStat object - * @throws S3ClientServiceException if an S3 error occurred - * @throws IOException if an S3 error occurred - */ - S3ClientStat statObject(String bucket, String object) throws S3ClientServiceException, IOException; - - /** - * Read an object - * @param bucket Bucket name - * @param object Object path - * @param offset Start reading at this offset - * @return InputStream to the object - * @throws S3ClientServiceException if an S3 error occurred - * @throws IOException if an S3 error occurred - */ - InputStream getObject(String bucket, String object, long offset) throws S3ClientServiceException, IOException; - - /** - * Download an object - * @param bucket Bucket name - * @param object Object path - * @param filename Destination file - * @throws S3ClientServiceException if an S3 error occurred - * @throws IOException if an S3 error occurred - */ - void getObject(String bucket, String object, String filename) throws S3ClientServiceException, IOException; - -} diff --git a/src/main/java/loci/common/services/S3ClientServiceException.java b/src/main/java/loci/common/services/S3ClientServiceException.java deleted file mode 100644 index 2c2d4a80..00000000 --- a/src/main/java/loci/common/services/S3ClientServiceException.java +++ /dev/null @@ -1,67 +0,0 @@ -/* - * #%L - * S3 client stat object - * %% - * Copyright (C) 2019 Open Microscopy Environment: - * - Board of Regents of the University of Wisconsin-Madison - * - Glencoe Software, Inc. - * - University of Dundee - * %% - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions are met: - * - * 1. Redistributions of source code must retain the above copyright notice, - * this list of conditions and the following disclaimer. - * 2. Redistributions in binary form must reproduce the above copyright notice, - * this list of conditions and the following disclaimer in the documentation - * and/or other materials provided with the distribution. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" - * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE - * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE - * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDERS OR CONTRIBUTORS BE - * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR - * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF - * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS - * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN - * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) - * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE - * POSSIBILITY OF SUCH DAMAGE. - * #L% - */ - -package loci.common.services; - -/** - * Exception thrown when internal error specific to the S3 client is raised - */ -public class S3ClientServiceException extends ServiceException -{ - /** - * Constructor. - * @param message Error message. - */ - public S3ClientServiceException(String message) - { - super(message); - } - - /** - * Constructor. - * @param message Error message. - * @param cause Upstream exception. - */ - public S3ClientServiceException(String message, Throwable cause) - { - super(message, cause); - } - - /** - * Constructor. - * @param cause Upstream exception. - */ - public S3ClientServiceException(Throwable cause) - { - super(cause); - } -} diff --git a/src/main/java/loci/common/services/S3ClientServiceImpl.java b/src/main/java/loci/common/services/S3ClientServiceImpl.java deleted file mode 100644 index 3b12b04c..00000000 --- a/src/main/java/loci/common/services/S3ClientServiceImpl.java +++ /dev/null @@ -1,132 +0,0 @@ -/* - * #%L - * S3 client service - * %% - * Copyright (C) 2019 Open Microscopy Environment: - * - Board of Regents of the University of Wisconsin-Madison - * - Glencoe Software, Inc. - * - University of Dundee - * %% - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions are met: - * - * 1. Redistributions of source code must retain the above copyright notice, - * this list of conditions and the following disclaimer. - * 2. Redistributions in binary form must reproduce the above copyright notice, - * this list of conditions and the following disclaimer in the documentation - * and/or other materials provided with the distribution. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" - * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE - * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE - * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDERS OR CONTRIBUTORS BE - * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR - * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF - * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS - * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN - * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) - * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE - * POSSIBILITY OF SUCH DAMAGE. - * #L% - */ - -package loci.common.services; -import io.minio.MinioClient; -import io.minio.errors.MinioException; -import io.minio.ObjectStat; -import org.xmlpull.v1.XmlPullParserException; - -import java.io.IOException; -import java.io.InputStream; -import java.security.InvalidKeyException; -import java.security.NoSuchAlgorithmException; - -/** - * An S3 client - */ -public class S3ClientServiceImpl extends AbstractService implements S3ClientService { - - /** - * Minio client - */ - private MinioClient s3Client; - - /** - * Default constructor. - */ - public S3ClientServiceImpl() { - checkClassDependency(io.minio.MinioClient.class); - } - - // -- S3ClientService methods - - @Override - public void initialize(String server, int port, String accessKey, String secretKey, - String appName, String appVersion) - throws S3ClientServiceException { - try { - s3Client = new MinioClient(server, port, accessKey, secretKey); - s3Client.setAppInfo(appName, appVersion); - } catch (MinioException e) { - throw new S3ClientServiceException(e); - } - } - - @Override - public boolean bucketExists(String bucket) throws S3ClientServiceException, IOException { - try { - return s3Client.bucketExists(bucket); - } - catch ( - MinioException | - InvalidKeyException | - NoSuchAlgorithmException | - XmlPullParserException e) { - throw new S3ClientServiceException(e); - } - } - - @Override - public S3ClientStat statObject(String bucket, String object) throws S3ClientServiceException, IOException { - try { - ObjectStat mcstat = s3Client.statObject(bucket, object); - return new S3ClientStat(mcstat.length()); - } - catch ( - MinioException | - InvalidKeyException | - NoSuchAlgorithmException | - XmlPullParserException e) { - throw new S3ClientServiceException(e); - } - } - - @Override - public InputStream getObject(String bucket, String object, long offset) throws S3ClientServiceException, IOException { - try { - return s3Client.getObject(bucket, object, offset); - } - catch ( - InvalidKeyException | - MinioException | - NoSuchAlgorithmException | - XmlPullParserException e) { - throw new S3ClientServiceException(e); - } - } - - @Override - public void getObject(String bucket, String object, String filename) throws S3ClientServiceException, IOException { - try { - s3Client.getObject(bucket, object, filename); - } - catch ( - InvalidKeyException | - MinioException | - NoSuchAlgorithmException | - XmlPullParserException e) { - throw new S3ClientServiceException(e); - } - } - -} diff --git a/src/main/java/loci/common/services/S3ClientStat.java b/src/main/java/loci/common/services/S3ClientStat.java deleted file mode 100644 index feceb1b4..00000000 --- a/src/main/java/loci/common/services/S3ClientStat.java +++ /dev/null @@ -1,59 +0,0 @@ -/* - * #%L - * S3 client stat object - * %% - * Copyright (C) 2019 Open Microscopy Environment: - * - Board of Regents of the University of Wisconsin-Madison - * - Glencoe Software, Inc. - * - University of Dundee - * %% - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions are met: - * - * 1. Redistributions of source code must retain the above copyright notice, - * this list of conditions and the following disclaimer. - * 2. Redistributions in binary form must reproduce the above copyright notice, - * this list of conditions and the following disclaimer in the documentation - * and/or other materials provided with the distribution. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" - * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE - * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE - * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDERS OR CONTRIBUTORS BE - * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR - * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF - * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS - * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN - * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) - * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE - * POSSIBILITY OF SUCH DAMAGE. - * #L% - */ - -package loci.common.services; - -/** - * Object stat (attributes) information - */ -public class S3ClientStat { - /** - * Create stat object storing length of object - * @param length - */ - S3ClientStat(long length) { - this.length = length; - } - - /** - * Length (size) of object - */ - private final long length; - - /** - * Get length (size) of object - * @return length - */ - public long length() { - return this.length; - }; -} \ No newline at end of file diff --git a/src/test/java/loci/common/utests/LocationTest.java b/src/test/java/loci/common/utests/LocationTest.java index 26f1753a..2835efda 100644 --- a/src/test/java/loci/common/utests/LocationTest.java +++ b/src/test/java/loci/common/utests/LocationTest.java @@ -65,7 +65,6 @@ public class LocationTest { private enum LocalRemoteType { LOCAL, HTTP, - S3, }; private Location[] files; @@ -76,7 +75,6 @@ private enum LocalRemoteType { private String[] mode; private LocalRemoteType[] isRemote; private static boolean runHttpRemoteTests; - private static boolean runS3RemoteTests; // -- Setup methods -- @@ -105,18 +103,11 @@ public void setup() throws IOException { new Location("https://www.openmicroscopy.org/"), new Location("https://www.openmicroscopy.org/nonexisting"), new Location("https://www.openmicroscopy.org/nonexisting/:/+/symbols"), - new Location(hiddenFile), - new Location("s3+http://localhost:31836/bucket-dne"), - new Location("s3+http://localhost:31836/bioformats.test.public"), - new Location("s3+http://localhost:31836/bioformats.test.public/single-channel.ome.tiff"), - new Location("s3+http://localhost:31836/bioformats.test.private/single-channel.ome.tiff"), - new Location("s3+http://accesskey:secretkey@localhost:31836/bioformats.test.private/single-channel.ome.tiff") - }; + new Location(hiddenFile),}; rootFiles = new Location[] { new Location("/"), new Location("https://www.openmicroscopy.org"), - new Location("s3://s3.example.org"), }; exists = new boolean[] { @@ -192,25 +183,16 @@ public void setup() throws IOException { LocalRemoteType.HTTP, LocalRemoteType.HTTP, LocalRemoteType.LOCAL, - LocalRemoteType.S3, - LocalRemoteType.S3, - LocalRemoteType.S3, - LocalRemoteType.S3, - LocalRemoteType.S3, }; } @BeforeClass public void checkProperties() throws IOException { runHttpRemoteTests = TestUtilities.getPropValueInt("testng.runHttpRemoteTests") > 0; - runS3RemoteTests = TestUtilities.getPropValueInt("testng.runS3RemoteTests") > 0; if (!runHttpRemoteTests) { LOGGER.warn("HTTP tests are disabled!"); } - if (!runS3RemoteTests) { - LOGGER.warn("S3 tests are disabled!"); - } } private void skipIfHttpDisabled(int i) throws SkipException { @@ -219,12 +201,6 @@ private void skipIfHttpDisabled(int i) throws SkipException { } } - private void skipIfS3Disabled(int i) throws SkipException { - if (isRemote[i] == LocalRemoteType.S3 && !runS3RemoteTests) { - throw new SkipException("S3 tests are disabled " + files[i].getName()); - } - } - // -- Tests -- // Order of assertEquals parameters is assertEquals(message, expected, actual) @@ -232,7 +208,6 @@ private void skipIfS3Disabled(int i) throws SkipException { public void testReadWriteMode() { for (int i=0; i 0; - - if (!runS3RemoteTests) { - LOGGER.warn("S3 tests are disabled!"); - } - } - - @BeforeMethod - public void setupMethod() throws S3ClientServiceException, SkipException { - if (!runS3RemoteTests) { - throw new SkipException("S3 tests are disabled"); - } - s3 = new S3ClientServiceImpl(); - s3.initialize("http://localhost", 31836, null, null, "S3ClientServiceTest", "0.0.0"); - } - - // -- Test methods -- - - @Test - public void testServiceLookup() throws DependencyException { - ServiceFactory factory = new ServiceFactory(); - S3ClientService s3client = factory.getInstance(S3ClientService.class); - assertTrue(S3ClientServiceImpl.class.isInstance(s3client)); - } - - @Test - public void testBucketExists() throws S3ClientServiceException, IOException { - assertTrue(s3.bucketExists("bioformats.test.public")); - } - - @Test(expectedExceptions = { S3ClientServiceException.class }) - public void testBucketNotExists() throws S3ClientServiceException, IOException { - s3.bucketExists("this.bucket.does.not.exist"); - } - - @Test - public void testStatObject() throws S3ClientServiceException, IOException { - S3ClientStat stat = s3.statObject("bioformats.test.public", "2MBfile.txt"); - assertEquals(2097152, stat.length()); - } - - @Test - public void testGetObject() throws S3ClientServiceException, IOException { - InputStream in = s3.getObject("bioformats.test.public", "2MBfile.txt", 1380896); - String line = new BufferedReader(new InputStreamReader(in)).readLine(); - assertEquals(". 43154", line); - in.close(); - } - - @Test - public void testGetObjectFile() throws S3ClientServiceException, IOException { - final String filepath = TEMPDIR + "/2MBfile.txt"; - s3.getObject("bioformats.test.public", "2MBfile.txt", filepath); - Path path = Paths.get(filepath); - assertEquals(2097152, Files.size(path)); - String[] lines = Files.newBufferedReader(path).lines().toArray(String[]::new); - assertEquals(". 1", lines[0]); - assertEquals(". 65536", lines[lines.length - 1]); - } - -} \ No newline at end of file diff --git a/src/test/java/loci/common/utests/S3HandleTest.java b/src/test/java/loci/common/utests/S3HandleTest.java deleted file mode 100644 index fc49416f..00000000 --- a/src/test/java/loci/common/utests/S3HandleTest.java +++ /dev/null @@ -1,301 +0,0 @@ -/* - * #%L - * Common package for I/O and related utilities - * %% - * Copyright (C) 2005 - 2016 Open Microscopy Environment: - * - Board of Regents of the University of Wisconsin-Madison - * - Glencoe Software, Inc. - * - University of Dundee - * %% - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions are met: - * - * 1. Redistributions of source code must retain the above copyright notice, - * this list of conditions and the following disclaimer. - * 2. Redistributions in binary form must reproduce the above copyright notice, - * this list of conditions and the following disclaimer in the documentation - * and/or other materials provided with the distribution. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" - * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE - * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE - * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDERS OR CONTRIBUTORS BE - * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR - * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF - * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS - * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN - * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) - * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE - * POSSIBILITY OF SUCH DAMAGE. - * #L% - */ - -package loci.common.utests; - -import loci.common.S3Handle; -import loci.common.StreamHandle; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import org.testng.SkipException; -import org.testng.annotations.BeforeClass; -import org.testng.annotations.Test; - -import java.io.IOException; -import java.nio.file.Files; -import java.nio.file.Path; -import java.nio.file.Paths; - -import static org.testng.AssertJUnit.assertEquals; -import static org.testng.AssertJUnit.assertFalse; -import static org.testng.AssertJUnit.assertTrue; - -/** - * Unit tests for the loci.common.S3Handle class. - * - * @see loci.common.URLHandle - */ -@Test(groups="readTests") -public class S3HandleTest { - - // -- Fields -- - - private static boolean runS3RemoteTests; - private Path TEMPDIR; - - private static final Logger LOGGER = LoggerFactory.getLogger(S3HandleTest.class); - - private static final String s3public = "s3+http://localhost:31836"; - private static final String s3private = "s3+http://accesskey:secretkey@localhost:31836"; - - // -- Setup methods -- - - @BeforeClass - public void setup() throws IOException { - TEMPDIR = Files.createTempDirectory("S3HandleTest-"); - TEMPDIR.toFile().deleteOnExit(); - - runS3RemoteTests = TestUtilities.getPropValueInt("testng.runS3RemoteTests") > 0; - - if (!runS3RemoteTests) { - LOGGER.warn("S3 tests are disabled!"); - } - } - - private void skipIfS3Disabled() throws SkipException { - if (!runS3RemoteTests) { - throw new SkipException("S3 tests are disabled"); - } - } - - // -- Test methods -- - - @Test - public void testCanHandleScheme() { - assertTrue(S3Handle.canHandleScheme("s3://")); - assertTrue(S3Handle.canHandleScheme("s3+transport://abc")); - assertTrue(S3Handle.canHandleScheme("s3+transport")); - assertFalse(S3Handle.canHandleScheme("s345://")); - assertFalse(S3Handle.canHandleScheme("http+s3://")); - assertFalse(S3Handle.canHandleScheme("https")); - } - - @Test - public void testParseLocalhost() throws IOException { - S3Handle s3 = new S3Handle("s3://localhost:9000/bucket/key/file.tif", false, null); - assertEquals("https://localhost", s3.getServer()); - assertEquals(9000, s3.getPort()); - assertEquals("bucket", s3.getBucket()); - assertEquals("key/file.tif", s3.getPath()); - } - - @Test - public void testParseAuth() throws IOException { - S3Handle s3 = new S3Handle( - "s3://access:secret@s3.example.org/bucket/key/file.tif", false, null); - assertEquals("https://s3.example.org", s3.getServer()); - assertEquals(0, s3.getPort()); - assertEquals("bucket", s3.getBucket()); - assertEquals("key/file.tif", s3.getPath()); - } - - @Test - public void testParseAuthLocalhost() throws IOException { - S3Handle s3 = new S3Handle("s3://access:secret@localhost:9000/bucket/key/file.tif", false, null); - assertEquals("https://localhost", s3.getServer()); - assertEquals(9000, s3.getPort()); - assertEquals("bucket", s3.getBucket()); - assertEquals("key/file.tif", s3.getPath()); - } - - @Test - public void testParseProtocol() throws IOException { - S3Handle s3 = new S3Handle( - "example://localhost/bucket/key/file.tif", false, null); - assertEquals("example://localhost", s3.getServer()); - assertEquals(0, s3.getPort()); - assertEquals("bucket", s3.getBucket()); - assertEquals("key/file.tif", s3.getPath()); - } - - @Test - public void testDefaultProtocol() throws IOException { - S3Handle s3 = new S3Handle("s3+custom://localhost/bucket/key/file.tif", false, null); - assertEquals("custom://localhost", s3.getServer()); - assertEquals(0, s3.getPort()); - assertEquals("bucket", s3.getBucket()); - assertEquals("key/file.tif", s3.getPath()); - } - - @Test - public void testParseNoSlash() throws IOException { - S3Handle s3 = new S3Handle("s3://localhost", false, null); - assertEquals("https://localhost", s3.getServer()); - assertEquals(0, s3.getPort()); - assertEquals(null, s3.getBucket()); - assertEquals(null, s3.getPath()); - } - - @Test - public void testParseSlashNoBucket() throws IOException { - S3Handle s3 = new S3Handle("s3://localhost/", false, null); - assertEquals("https://localhost", s3.getServer()); - assertEquals(0, s3.getPort()); - assertEquals(null, s3.getBucket()); - assertEquals(null, s3.getPath()); - } - - @Test - public void testParseBucketNoSlash() throws IOException { - S3Handle s3 = new S3Handle("s3://localhost/bucket", false, null); - assertEquals("https://localhost", s3.getServer()); - assertEquals(0, s3.getPort()); - assertEquals("bucket", s3.getBucket()); - assertEquals(null, s3.getPath()); - } - - @Test - public void testParseBucketSlash() throws IOException { - S3Handle s3 = new S3Handle("s3://localhost/bucket/", false, null); - assertEquals("https://localhost", s3.getServer()); - assertEquals(0, s3.getPort()); - assertEquals("bucket", s3.getBucket()); - assertEquals(null, s3.getPath()); - } - - @Test - public void testIsBucket() throws IOException { - skipIfS3Disabled(); - S3Handle s3 = new S3Handle(s3public + "/bioformats.test.public"); - assertTrue(s3.isBucket()); - } - - @Test - public void testReadPublic() throws IOException { - skipIfS3Disabled(); - S3Handle s3 = new S3Handle(s3public + "/bioformats.test.public/single-channel.ome.tiff"); - assertFalse(s3.isBucket()); - assertTrue(s3.exists()); - assertEquals(76097, s3.length()); - } - - @Test - public void testReadPrivate() throws IOException { - skipIfS3Disabled(); - S3Handle s3 = new S3Handle(s3private + "/bioformats.test.private/single-channel.ome.tiff"); - assertFalse(s3.isBucket()); - assertTrue(s3.exists()); - assertEquals(76097, s3.length()); - } - - @Test - public void testReadAndSeek() throws IOException { - skipIfS3Disabled(); - S3Handle s3 = new S3Handle(s3public + "/bioformats.test.public/2MBfile.txt"); - assertFalse(s3.isBucket()); - assertTrue(s3.exists()); - assertEquals(2097152, s3.length()); - - byte[] buffer = new byte[32]; - int r; - - r = s3.read(buffer, 0, 32); - assertEquals(32, r); - assertEquals(". 1\n", new String(buffer)); - - r = s3.read(buffer, 0, 32); - assertEquals(32, r); - assertEquals(". 2\n", new String(buffer)); - - s3.seek(80); - r = s3.read(buffer, 0, 32); - assertEquals(32, r); - assertEquals(" 3\n. ", new String(buffer)); - - // Large seek (S3Handle.S3_MAX_FORWARD_SEEK) - s3.seek(2097056); - r = s3.read(buffer, 0, 32); - assertEquals(32, r); - assertEquals(". 65534\n", new String(buffer)); - - // Reverse seek - s3.seek(144); - r = s3.read(buffer, 0, 32); - assertEquals(32, r); - assertEquals(" 5\n. ", new String(buffer)); - } - - @Test - public void testResetStream() throws IOException { - class S3HandleWrapper extends S3Handle { - public S3HandleWrapper(String url) throws IOException { - super(url); - } - - @Override - public void resetStream() throws IOException { - super.resetStream(); - } - - @Override - public void resetStream(long offset) throws IOException { - super.resetStream(offset); - } - } - - skipIfS3Disabled(); - S3HandleWrapper s3 = new S3HandleWrapper(s3private + "/bioformats.test.private/2MBfile.txt"); - assertFalse(s3.isBucket()); - assertTrue(s3.exists()); - assertEquals(2097152, s3.length()); - - byte[] buffer = new byte[32]; - int r; - s3.resetStream(750144); - r = s3.read(buffer, 0, 32); - assertEquals(32, r); - assertEquals(". 23443\n", new String(buffer)); - - s3.resetStream(); - r = s3.read(buffer, 0, 32); - assertEquals(32, r); - assertEquals(". 1\n", new String(buffer)); - } - - @Test - public void testCache() throws IOException { - class MockSettings extends StreamHandle.Settings { - @Override - public String getRemoteCacheRootDir() { - return TEMPDIR.toString(); - } - } - - skipIfS3Disabled(); - final String expectedPath = TEMPDIR + "/http/localhost/31836/bioformats.test.public/2MBfile.txt"; - - String downloaded = S3Handle.cacheObject( - s3public + "/bioformats.test.public/2MBfile.txt", new MockSettings()); - assertEquals(expectedPath, downloaded); - assertEquals(2097152, Files.size(Paths.get(downloaded))); - } -} \ No newline at end of file