diff --git a/ndbench-dax-plugins/src/main/java/com/netflix/ndbench/plugin/dax/DaxKeyValue.java b/ndbench-dax-plugins/src/main/java/com/netflix/ndbench/plugin/dax/DaxKeyValue.java index 49b0b602..4fb09732 100644 --- a/ndbench-dax-plugins/src/main/java/com/netflix/ndbench/plugin/dax/DaxKeyValue.java +++ b/ndbench-dax-plugins/src/main/java/com/netflix/ndbench/plugin/dax/DaxKeyValue.java @@ -18,27 +18,22 @@ import com.amazon.dax.client.dynamodbv2.AmazonDaxClientBuilder; import com.amazonaws.auth.AWSCredentialsProvider; -import com.amazonaws.services.dynamodbv2.model.ReturnConsumedCapacity; -import com.google.common.base.Preconditions; +import com.amazonaws.services.dynamodbv2.AmazonDynamoDB; + import com.google.inject.Inject; import com.google.inject.Singleton; -import com.netflix.ndbench.api.plugin.DataGenerator; import com.netflix.ndbench.api.plugin.NdBenchClient; import com.netflix.ndbench.api.plugin.annotations.NdBenchClientPlugin; -import com.netflix.ndbench.plugin.dynamodb.DynamoDBKeyValueBase; import com.netflix.ndbench.plugin.dax.configs.DaxConfiguration; -import com.netflix.ndbench.plugin.dynamodb.operations.dynamodb.dataplane.DynamoDBReadBulk; -import com.netflix.ndbench.plugin.dynamodb.operations.dynamodb.dataplane.DynamoDBReadSingle; -import com.netflix.ndbench.plugin.dynamodb.operations.dynamodb.dataplane.DynamoDBWriteBulk; -import com.netflix.ndbench.plugin.dynamodb.operations.dynamodb.dataplane.DynamoDBWriteSingle; -import org.apache.commons.lang.StringUtils; +import com.netflix.ndbench.plugin.dynamodb.BaseConfigurationDynamoDBKeyValue; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @Singleton @NdBenchClientPlugin("DaxKeyValue") -public class DaxKeyValue extends DynamoDBKeyValueBase implements NdBenchClient { +public class DaxKeyValue extends BaseConfigurationDynamoDBKeyValue implements NdBenchClient { private static final Logger logger = LoggerFactory.getLogger(DaxKeyValue.class); + private AmazonDynamoDB dynamoDB; /** * Public constructor to inject credentials and configuration @@ -52,31 +47,15 @@ public DaxKeyValue(AWSCredentialsProvider awsCredentialsProvider, DaxConfigurati } @Override - public void init(DataGenerator dataGenerator) { - logger.info("Initializing AWS DAX client"); - + protected void createAndSetDynamoDBClient() { AmazonDaxClientBuilder amazonDaxClientBuilder = AmazonDaxClientBuilder.standard(); amazonDaxClientBuilder.withEndpointConfiguration(this.config.getDaxEndpoint()); dynamoDB = amazonDaxClientBuilder.build(); + } - //instantiate operations - String tableName = config.getTableName(); - String partitionKeyName = config.getAttributeName(); - ReturnConsumedCapacity returnConsumedCapacity = ReturnConsumedCapacity.NONE; - Preconditions.checkState(StringUtils.isNotEmpty(tableName)); - Preconditions.checkState(StringUtils.isNotEmpty(partitionKeyName)); - - //data plane - boolean consistentRead = config.consistentRead(); - this.singleRead = new DynamoDBReadSingle(dataGenerator, dynamoDB, tableName, partitionKeyName, consistentRead, - returnConsumedCapacity); - this.bulkRead = new DynamoDBReadBulk(dataGenerator, dynamoDB, tableName, partitionKeyName, consistentRead, - returnConsumedCapacity); - this.singleWrite = new DynamoDBWriteSingle(dataGenerator, dynamoDB, tableName, partitionKeyName, - returnConsumedCapacity); - this.bulkWrite = new DynamoDBWriteBulk(dataGenerator, dynamoDB, tableName, partitionKeyName, - returnConsumedCapacity); - - logger.info("DynamoDB Plugin initialized"); + @Override + public void shutdown() { + dynamoDB.shutdown(); + logger.info("Shutdown DynamoDB."); } } diff --git a/ndbench-dynamodb-plugins/build.gradle b/ndbench-dynamodb-plugins/build.gradle index 48d6edae..654f4f09 100644 --- a/ndbench-dynamodb-plugins/build.gradle +++ b/ndbench-dynamodb-plugins/build.gradle @@ -35,6 +35,10 @@ dependencies { compile "com.amazonaws:aws-java-sdk-applicationautoscaling:latest.release" compile "com.amazonaws:aws-java-sdk-sts:latest.release" compile "com.amazonaws:aws-java-sdk-cloudwatch:latest.release" + compile "software.amazon.awssdk:dynamodb:latest.release" + compile "software.amazon.awssdk:applicationautoscaling:latest.release" + compile "software.amazon.awssdk:sts:latest.release" + compile "software.amazon.awssdk:cloudwatch:latest.release" testCompile group: 'com.netflix.archaius', name: 'archaius2-guice', version: '2.1.11' testCompile 'com.netflix.governator:governator-test-junit:[1.15.3,)' testCompile project(':ndbench-core') diff --git a/ndbench-dynamodb-plugins/src/main/java/com/netflix/ndbench/plugin/dynamodb/BaseConfigurationDynamoDBKeyValue.java b/ndbench-dynamodb-plugins/src/main/java/com/netflix/ndbench/plugin/dynamodb/BaseConfigurationDynamoDBKeyValue.java new file mode 100644 index 00000000..5a77ceee --- /dev/null +++ b/ndbench-dynamodb-plugins/src/main/java/com/netflix/ndbench/plugin/dynamodb/BaseConfigurationDynamoDBKeyValue.java @@ -0,0 +1,135 @@ +/* + * Copyright 2018 Netflix, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ +package com.netflix.ndbench.plugin.dynamodb; + +import com.amazonaws.ClientConfiguration; +import com.amazonaws.client.builder.AwsClientBuilder; +import com.amazonaws.retry.RetryPolicy; +import com.amazonaws.services.dynamodbv2.model.BatchGetItemResult; +import com.amazonaws.services.dynamodbv2.model.BatchWriteItemResult; +import com.amazonaws.services.dynamodbv2.model.GetItemResult; +import com.amazonaws.services.dynamodbv2.model.PutItemResult; +import com.google.common.base.Preconditions; + +import com.amazonaws.services.dynamodbv2.model.ReturnConsumedCapacity; +import com.netflix.ndbench.plugin.dynamodb.configs.DynamoDBConfigurationBase; +import com.netflix.ndbench.plugin.dynamodb.operations.v1.dynamodb.dataplane.DynamoDBReadBulk; +import com.netflix.ndbench.plugin.dynamodb.operations.v1.dynamodb.dataplane.DynamoDBReadSingle; +import com.netflix.ndbench.plugin.dynamodb.operations.v1.dynamodb.dataplane.DynamoDBWriteBulk; +import com.netflix.ndbench.plugin.dynamodb.operations.v1.dynamodb.dataplane.DynamoDBWriteSingle; + +import org.apache.commons.lang.StringUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.amazonaws.auth.AWSCredentialsProvider; +import com.amazonaws.services.dynamodbv2.AmazonDynamoDB; +import com.amazonaws.services.dynamodbv2.AmazonDynamoDBClientBuilder; +import com.netflix.ndbench.api.plugin.DataGenerator; +import com.netflix.ndbench.api.plugin.NdBenchClient; + +import static com.amazonaws.retry.PredefinedRetryPolicies.DEFAULT_RETRY_CONDITION; +import static com.amazonaws.retry.PredefinedRetryPolicies.DYNAMODB_DEFAULT_BACKOFF_STRATEGY; +import static com.amazonaws.retry.PredefinedRetryPolicies.NO_RETRY_POLICY; + +/** + * This NDBench plugin provides a single key value for AWS DynamoDB. + * + * @author ipapapa + * @author Alexander Patrikalakis + */ +public abstract class BaseConfigurationDynamoDBKeyValue extends DynamoDBKeyValueBase implements NdBenchClient { + private static final boolean DO_HONOR_MAX_ERROR_RETRY_IN_CLIENT_CONFIG = true; + private static final Logger logger = LoggerFactory.getLogger(BaseConfigurationDynamoDBKeyValue.class); + + AmazonDynamoDB dynamoDB; + + /** + * Public constructor to inject credentials and configuration + * + * @param awsCredentialsProvider + * @param configuration + */ + + public BaseConfigurationDynamoDBKeyValue(AWSCredentialsProvider awsCredentialsProvider, C configuration) { + super(awsCredentialsProvider, configuration); + } + + protected void createAndSetDynamoDBClient() { + AmazonDynamoDBClientBuilder dynamoDbBuilder = AmazonDynamoDBClientBuilder.standard(); + dynamoDbBuilder.withClientConfiguration(new ClientConfiguration() + .withMaxConnections(config.getMaxConnections()) + .withRequestTimeout(config.getMaxRequestTimeout()) //milliseconds + .withRetryPolicy(config.getMaxRetries() <= 0 ? NO_RETRY_POLICY : new RetryPolicy(DEFAULT_RETRY_CONDITION, + DYNAMODB_DEFAULT_BACKOFF_STRATEGY, + config.getMaxRetries(), + DO_HONOR_MAX_ERROR_RETRY_IN_CLIENT_CONFIG)) + .withGzip(config.isCompressing())); + dynamoDbBuilder.withCredentials(awsCredentialsProvider); + if (StringUtils.isNotEmpty(this.config.getEndpoint())) { + Preconditions.checkState(StringUtils.isNotEmpty(config.getRegion()), + "If you set the endpoint you must set the region"); + dynamoDbBuilder.withEndpointConfiguration( + new AwsClientBuilder.EndpointConfiguration(config.getEndpoint(), config.getRegion())); + } + this.dynamoDB = dynamoDbBuilder.build(); + } + + protected void instantiateDataPlaneOperations(DataGenerator dataGenerator) { + //instantiate operations + String tableName = config.getTableName(); + String partitionKeyName = config.getAttributeName(); + ReturnConsumedCapacity returnConsumedCapacity = ReturnConsumedCapacity.NONE; + Preconditions.checkState(StringUtils.isNotEmpty(tableName)); + Preconditions.checkState(StringUtils.isNotEmpty(partitionKeyName)); + + //data plane + boolean consistentRead = config.consistentRead(); + this.singleRead = new DynamoDBReadSingle(dataGenerator, dynamoDB, tableName, partitionKeyName, consistentRead, + returnConsumedCapacity); + this.bulkRead = new DynamoDBReadBulk(dataGenerator, dynamoDB, tableName, partitionKeyName, consistentRead, + returnConsumedCapacity); + this.singleWrite = new DynamoDBWriteSingle(dataGenerator, dynamoDB, tableName, partitionKeyName, + returnConsumedCapacity); + this.bulkWrite = new DynamoDBWriteBulk(dataGenerator, dynamoDB, tableName, partitionKeyName, + returnConsumedCapacity); + } + + @Override + public void shutdown() { + logger.info("DynamoDB shutdown"); + dynamoDB.shutdown(); + } + + /* + * Not needed for this plugin + * + * @see com.netflix.ndbench.api.plugin.NdBenchClient#getConnectionInfo() + */ + @Override + public String getConnectionInfo() { + return String.format("Table Name - %s : Attribute Name - %s : Consistent Read - %b", + this.config.getTableName(), + this.config.getAttributeName(), + this.config.consistentRead()); + } + + AmazonDynamoDB getDynamoDB() { + return this.dynamoDB; + } +} diff --git a/ndbench-dynamodb-plugins/src/main/java/com/netflix/ndbench/plugin/dynamodb/DynamoDBKeyValue.java b/ndbench-dynamodb-plugins/src/main/java/com/netflix/ndbench/plugin/dynamodb/DynamoDBKeyValue.java index df2ebc39..d60b6b6a 100644 --- a/ndbench-dynamodb-plugins/src/main/java/com/netflix/ndbench/plugin/dynamodb/DynamoDBKeyValue.java +++ b/ndbench-dynamodb-plugins/src/main/java/com/netflix/ndbench/plugin/dynamodb/DynamoDBKeyValue.java @@ -16,30 +16,21 @@ */ package com.netflix.ndbench.plugin.dynamodb; -import com.netflix.ndbench.plugin.dynamodb.configs.DynamoDBConfiguration; - -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - import com.amazonaws.auth.AWSCredentialsProvider; -import com.amazonaws.services.dynamodbv2.AmazonDynamoDB; import com.google.inject.Inject; import com.google.inject.Singleton; -import com.netflix.ndbench.api.plugin.DataGenerator; -import com.netflix.ndbench.api.plugin.NdBenchClient; import com.netflix.ndbench.api.plugin.annotations.NdBenchClientPlugin; +import com.netflix.ndbench.plugin.dynamodb.configs.DynamoDBConfiguration; /** * This NDBench plugin provides a single key value for AWS DynamoDB. - * + * * @author ipapapa * @author Alexander Patrikalakis */ @Singleton @NdBenchClientPlugin("DynamoDBKeyValue") -public class DynamoDBKeyValue extends DynamoDBKeyValueBase implements NdBenchClient { - private static final Logger logger = LoggerFactory.getLogger(DynamoDBKeyValue.class); - +public class DynamoDBKeyValue extends BaseConfigurationDynamoDBKeyValue { /** * Public constructor to inject credentials and configuration * @@ -50,31 +41,4 @@ public class DynamoDBKeyValue extends DynamoDBKeyValueBase implements NdBenchClient { +public abstract class DynamoDBKeyValueBase implements NdBenchClient { private static final Logger logger = LoggerFactory.getLogger(DynamoDBKeyValueBase.class); - private static final boolean DO_HONOR_MAX_ERROR_RETRY_IN_CLIENT_CONFIG = true; - - protected final AWSCredentialsProvider awsCredentialsProvider; + protected final CredentialsProviderType awsCredentialsProvider; protected final C config; // dynamically initialized - protected AmazonDynamoDB dynamoDB; - protected DynamoDBReadSingle singleRead; - protected DynamoDBReadBulk bulkRead; - protected DynamoDBWriteSingle singleWrite; - protected DynamoDBWriteBulk bulkWrite; + protected CapacityConsumingFunction singleRead; + protected CapacityConsumingFunction, List> bulkRead; + protected CapacityConsumingFunction singleWrite; + protected CapacityConsumingFunction, List> bulkWrite; /** * Protected method to inject credentials and configuration * @param awsCredentialsProvider * @param configuration */ - protected DynamoDBKeyValueBase(AWSCredentialsProvider awsCredentialsProvider, + protected DynamoDBKeyValueBase(CredentialsProviderType awsCredentialsProvider, C configuration) { this.awsCredentialsProvider = awsCredentialsProvider; this.config = configuration; } - protected void instantiateDataPlaneOperations(DataGenerator dataGenerator) { - //instantiate operations - String tableName = config.getTableName(); - String partitionKeyName = config.getAttributeName(); - ReturnConsumedCapacity returnConsumedCapacity = ReturnConsumedCapacity.NONE; - Preconditions.checkState(StringUtils.isNotEmpty(tableName)); - Preconditions.checkState(StringUtils.isNotEmpty(partitionKeyName)); - - //data plane - boolean consistentRead = config.consistentRead(); - this.singleRead = new DynamoDBReadSingle(dataGenerator, dynamoDB, tableName, partitionKeyName, consistentRead, - returnConsumedCapacity); - this.bulkRead = new DynamoDBReadBulk(dataGenerator, dynamoDB, tableName, partitionKeyName, consistentRead, - returnConsumedCapacity); - this.singleWrite = new DynamoDBWriteSingle(dataGenerator, dynamoDB, tableName, partitionKeyName, - returnConsumedCapacity); - this.bulkWrite = new DynamoDBWriteBulk(dataGenerator, dynamoDB, tableName, partitionKeyName, - returnConsumedCapacity); - } + protected abstract void instantiateDataPlaneOperations(DataGenerator dataGenerator); - protected void createAndSetDynamoDBClient() { - AmazonDynamoDBClientBuilder dynamoDbBuilder = AmazonDynamoDBClientBuilder.standard(); - dynamoDbBuilder.withClientConfiguration(new ClientConfiguration() - .withMaxConnections(config.getMaxConnections()) - .withRequestTimeout(config.getMaxRequestTimeout()) //milliseconds - .withRetryPolicy(config.getMaxRetries() <= 0 ? NO_RETRY_POLICY : new RetryPolicy(DEFAULT_RETRY_CONDITION, - DYNAMODB_DEFAULT_BACKOFF_STRATEGY, - config.getMaxRetries(), - DO_HONOR_MAX_ERROR_RETRY_IN_CLIENT_CONFIG)) - .withGzip(config.isCompressing())); - dynamoDbBuilder.withCredentials(awsCredentialsProvider); - if (StringUtils.isNotEmpty(this.config.getEndpoint())) { - Preconditions.checkState(StringUtils.isNotEmpty(config.getRegion()), - "If you set the endpoint you must set the region"); - dynamoDbBuilder.withEndpointConfiguration( - new AwsClientBuilder.EndpointConfiguration(config.getEndpoint(), config.getRegion())); - } - this.dynamoDB = dynamoDbBuilder.build(); - } + protected abstract void createAndSetDynamoDBClient(); @Override - public abstract void init(DataGenerator dataGenerator); + public void init(DataGenerator dataGenerator) { + logger.info("Initializing AWS SDK clients"); + createAndSetDynamoDBClient(); + instantiateDataPlaneOperations(dataGenerator); + logger.info("DynamoDB Plugin initialized"); + } @Override public String readSingle(String key){ @@ -132,10 +89,8 @@ public List writeBulk(List keys) { return bulkWrite.apply(keys); } - public void shutdown() { - dynamoDB.shutdown(); - logger.info("DynamoDB shutdown"); - } + @Override + public abstract void shutdown(); @Override public String getConnectionInfo() { diff --git a/ndbench-dynamodb-plugins/src/main/java/com/netflix/ndbench/plugin/dynamodb/DynamoDBProgrammaticKeyValue.java b/ndbench-dynamodb-plugins/src/main/java/com/netflix/ndbench/plugin/dynamodb/DynamoDBProgrammaticKeyValue.java index 98ca489b..916c7099 100644 --- a/ndbench-dynamodb-plugins/src/main/java/com/netflix/ndbench/plugin/dynamodb/DynamoDBProgrammaticKeyValue.java +++ b/ndbench-dynamodb-plugins/src/main/java/com/netflix/ndbench/plugin/dynamodb/DynamoDBProgrammaticKeyValue.java @@ -36,11 +36,11 @@ import com.netflix.ndbench.api.plugin.NdBenchClient; import com.netflix.ndbench.api.plugin.annotations.NdBenchClientPlugin; import com.netflix.ndbench.plugin.dynamodb.configs.ProgrammaticDynamoDBConfiguration; -import com.netflix.ndbench.plugin.dynamodb.operations.cloudwatch.controlplane.PutMetricAlarmOperation; -import com.netflix.ndbench.plugin.dynamodb.operations.cloudwatch.dataplane.PutMetricDataOperation; -import com.netflix.ndbench.plugin.dynamodb.operations.dynamodb.controlplane.DescribeLimits; -import com.netflix.ndbench.plugin.dynamodb.operations.dynamodb.controlplane.CreateDynamoDBTable; -import com.netflix.ndbench.plugin.dynamodb.operations.dynamodb.controlplane.DeleteDynamoDBTable; +import com.netflix.ndbench.plugin.dynamodb.operations.v1.cloudwatch.controlplane.PutMetricAlarmOperation; +import com.netflix.ndbench.plugin.dynamodb.operations.v1.cloudwatch.dataplane.PutMetricDataOperation; +import com.netflix.ndbench.plugin.dynamodb.operations.v1.dynamodb.controlplane.DescribeLimits; +import com.netflix.ndbench.plugin.dynamodb.operations.v1.dynamodb.controlplane.CreateDynamoDBTable; +import com.netflix.ndbench.plugin.dynamodb.operations.v1.dynamodb.controlplane.DeleteDynamoDBTable; import org.apache.commons.lang.StringUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -67,7 +67,7 @@ */ @Singleton @NdBenchClientPlugin("DynamoDBProgrammaticKeyValue") -public class DynamoDBProgrammaticKeyValue extends DynamoDBKeyValueBase implements NdBenchClient { +public class DynamoDBProgrammaticKeyValue extends BaseConfigurationDynamoDBKeyValue implements NdBenchClient { private static final Logger logger = LoggerFactory.getLogger(DynamoDBProgrammaticKeyValue.class); private static final String ND_BENCH_DYNAMO_DB_CONSUMED_RCU = "ConsumedRcuHighRes"; private static final String ND_BENCH_DYNAMO_DB_CONSUMED_WCU = "ConsumedWcuHighRes"; @@ -101,8 +101,7 @@ public DynamoDBProgrammaticKeyValue(AWSCredentialsProvider awsCredentialsProvide @Override public void init(DataGenerator dataGenerator) { - createAndSetDynamoDBClient(); - instantiateDataPlaneOperations(dataGenerator); + super.init(dataGenerator); //prerequisite data from configuration String tableName = config.getTableName(); @@ -222,7 +221,6 @@ private MetricDatum createCapacityUnitMetricDatumAndResetCounter(Date now, doubl @Override public void shutdown() { - super.shutdown(); if (cloudwatchReporterExecutor.get() != null) { cloudwatchReporterExecutor.get().shutdownNow(); cloudwatchReporterExecutor.set(null); @@ -230,5 +228,6 @@ public void shutdown() { cloudWatch.shutdown(); logger.info("CloudWatch shutdown"); deleteTable.delete(); + super.shutdown(); } } diff --git a/ndbench-dynamodb-plugins/src/main/java/com/netflix/ndbench/plugin/dynamodb/DynamoDBv2KeyValue.java b/ndbench-dynamodb-plugins/src/main/java/com/netflix/ndbench/plugin/dynamodb/DynamoDBv2KeyValue.java new file mode 100644 index 00000000..42981b95 --- /dev/null +++ b/ndbench-dynamodb-plugins/src/main/java/com/netflix/ndbench/plugin/dynamodb/DynamoDBv2KeyValue.java @@ -0,0 +1,145 @@ +/* + * Copyright 2018 Netflix, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ +package com.netflix.ndbench.plugin.dynamodb; + +import java.net.URISyntaxException; + +import com.google.common.base.Preconditions; + +import com.google.common.base.Strings; +import com.netflix.ndbench.plugin.dynamodb.configs.DynamoDBConfiguration; + +import com.netflix.ndbench.plugin.dynamodb.operations.v2.dynamodb.dataplane.DynamoDBReadBulk; +import com.netflix.ndbench.plugin.dynamodb.operations.v2.dynamodb.dataplane.DynamoDBReadSingle; +import com.netflix.ndbench.plugin.dynamodb.operations.v2.dynamodb.dataplane.DynamoDBWriteBulk; +import com.netflix.ndbench.plugin.dynamodb.operations.v2.dynamodb.dataplane.DynamoDBWriteSingle; +import org.apache.commons.lang.StringUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.net.URI; + +import com.google.inject.Inject; +import com.google.inject.Singleton; +import com.netflix.ndbench.api.plugin.DataGenerator; +import com.netflix.ndbench.api.plugin.NdBenchClient; +import com.netflix.ndbench.api.plugin.annotations.NdBenchClientPlugin; +import software.amazon.awssdk.auth.credentials.AwsCredentialsProvider; +import software.amazon.awssdk.http.Protocol; +import software.amazon.awssdk.http.nio.netty.NettyNioAsyncHttpClient; +import software.amazon.awssdk.regions.Region; +import software.amazon.awssdk.services.dynamodb.DynamoDbAsyncClient; +import software.amazon.awssdk.services.dynamodb.DynamoDbAsyncClientBuilder; +import software.amazon.awssdk.services.dynamodb.model.BatchGetItemResponse; +import software.amazon.awssdk.services.dynamodb.model.BatchWriteItemResponse; +import software.amazon.awssdk.services.dynamodb.model.GetItemResponse; +import software.amazon.awssdk.services.dynamodb.model.PutItemResponse; +import software.amazon.awssdk.services.dynamodb.model.ReturnConsumedCapacity; + +/** + * This NDBench plugin provides a single key value for AWS DynamoDB. + * + * @author ipapapa + * @author Alexander Patrikalakis + */ +@Singleton +@NdBenchClientPlugin("DynamoDBv2KeyValue") +public class DynamoDBv2KeyValue extends DynamoDBKeyValueBase implements NdBenchClient { + private static final Logger logger = LoggerFactory.getLogger(DynamoDBv2KeyValue.class); + + // dynamically initialized + private DynamoDbAsyncClient dynamoDB; + + /** + * Protected method to inject credentials and configuration + * + * @param awsCredentialsProvider + * @param configuration + */ + @Inject + public DynamoDBv2KeyValue(AwsCredentialsProvider awsCredentialsProvider, DynamoDBConfiguration configuration) { + super(awsCredentialsProvider, configuration); + } + + @Override + protected void instantiateDataPlaneOperations(DataGenerator dataGenerator) { + //instantiate operations + ReturnConsumedCapacity returnConsumedCapacity = ReturnConsumedCapacity.NONE; + String tableName = config.getTableName(); + String partitionKeyName = config.getAttributeName(); + Preconditions.checkState(StringUtils.isNotEmpty(tableName)); + Preconditions.checkState(StringUtils.isNotEmpty(partitionKeyName)); + + //data plane + boolean consistentRead = config.consistentRead(); + this.singleRead = new DynamoDBReadSingle(dataGenerator, dynamoDB, tableName, partitionKeyName, consistentRead, + returnConsumedCapacity); + this.bulkRead = new DynamoDBReadBulk(dataGenerator, dynamoDB, tableName, partitionKeyName, consistentRead, + returnConsumedCapacity); + this.singleWrite = new DynamoDBWriteSingle(dataGenerator, dynamoDB, tableName, partitionKeyName, + returnConsumedCapacity); + this.bulkWrite = new DynamoDBWriteBulk(dataGenerator, dynamoDB, tableName, partitionKeyName, + returnConsumedCapacity); + } + + @Override + protected void createAndSetDynamoDBClient() { + DynamoDbAsyncClientBuilder builder = DynamoDbAsyncClient.builder().credentialsProvider(awsCredentialsProvider); + //TODO figure out how to set request timeout and retry policy in AWS SDK 2.0 + builder.httpClient(NettyNioAsyncHttpClient.builder() + .maxConcurrency(config.getMaxConnections()) + .protocol(Protocol.HTTP1_1) //TODO evaluate HTTP2 + .build()); + + if (!Strings.isNullOrEmpty(this.config.getEndpoint())) { + Preconditions.checkState(!Strings.isNullOrEmpty(config.getRegion()), + "If you set the endpoint you must set the region"); + try { + builder.endpointOverride(new URI(config.getEndpoint())); + } catch (URISyntaxException e) { + throw new IllegalStateException("Unable to convert endpoint", e); + } + builder.region(Region.of(config.getRegion())); + } + dynamoDB = builder.build(); + } + + @Override + public void shutdown() { + dynamoDB.close(); + logger.info("DynamoDB shutdown"); + } + + /* + * Not needed for this plugin + * + * @see com.netflix.ndbench.api.plugin.NdBenchClient#getConnectionInfo() + */ + @Override + public String getConnectionInfo() { + return String.format("Table Name - %s : Attribute Name - %s : Consistent Read - %b", + this.config.getTableName(), + this.config.getAttributeName(), + this.config.consistentRead()); + } + + @Override + public String runWorkFlow() { + return null; + } +} diff --git a/ndbench-dynamodb-plugins/src/main/java/com/netflix/ndbench/plugin/dynamodb/NdbenchAWSCredentialProviderV2.java b/ndbench-dynamodb-plugins/src/main/java/com/netflix/ndbench/plugin/dynamodb/NdbenchAWSCredentialProviderV2.java new file mode 100644 index 00000000..df518821 --- /dev/null +++ b/ndbench-dynamodb-plugins/src/main/java/com/netflix/ndbench/plugin/dynamodb/NdbenchAWSCredentialProviderV2.java @@ -0,0 +1,62 @@ +/* + * Copyright 2018 Netflix, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ +package com.netflix.ndbench.plugin.dynamodb; + +import com.google.inject.Inject; +import com.google.inject.Singleton; +import com.netflix.ndbench.aws.config.CredentialsConfiguration; +import org.apache.commons.lang3.StringUtils; +import software.amazon.awssdk.auth.credentials.AwsBasicCredentials; +import software.amazon.awssdk.auth.credentials.AwsCredentials; +import software.amazon.awssdk.auth.credentials.AwsCredentialsProvider; +import software.amazon.awssdk.auth.credentials.DefaultCredentialsProvider; +import software.amazon.awssdk.auth.credentials.StaticCredentialsProvider; + +/** + * This credentials provider allows refreshing from Archaius configuration if the Archaius configuration contains + * credentials. Otherwise, this credentials provider delegates to the default AWS credentials provider chain. + * + * @author Alexander Patrikalakis + * @author ipapapa + */ +@Singleton +public class NdbenchAWSCredentialProviderV2 implements AwsCredentialsProvider { + private final CredentialsConfiguration config; + private volatile AwsCredentialsProvider credentialsProvider; + + @Inject + public NdbenchAWSCredentialProviderV2(CredentialsConfiguration config) { + this.config = config; + refresh(); + } + + private void refresh() { + if (StringUtils.isNotEmpty(config.accessKey()) && StringUtils.isNotEmpty(config.secretKey())) { + credentialsProvider + = StaticCredentialsProvider.create( + AwsBasicCredentials.create(config.accessKey(), config.secretKey())); + } else { + credentialsProvider = DefaultCredentialsProvider.create(); + } + } + + @Override + public AwsCredentials resolveCredentials() { + refresh(); + return credentialsProvider.resolveCredentials(); + } +} \ No newline at end of file diff --git a/ndbench-dynamodb-plugins/src/main/java/com/netflix/ndbench/plugin/dynamodb/configs/DynamoDBModule.java b/ndbench-dynamodb-plugins/src/main/java/com/netflix/ndbench/plugin/dynamodb/configs/DynamoDBModule.java index c110dfa7..53aa3491 100644 --- a/ndbench-dynamodb-plugins/src/main/java/com/netflix/ndbench/plugin/dynamodb/configs/DynamoDBModule.java +++ b/ndbench-dynamodb-plugins/src/main/java/com/netflix/ndbench/plugin/dynamodb/configs/DynamoDBModule.java @@ -16,10 +16,14 @@ */ package com.netflix.ndbench.plugin.dynamodb.configs; +import com.amazonaws.auth.AWSCredentialsProvider; import com.google.inject.AbstractModule; import com.google.inject.Provides; import com.netflix.archaius.ConfigProxyFactory; import com.netflix.ndbench.api.plugin.annotations.NdBenchClientPluginGuiceModule; +import com.netflix.ndbench.aws.config.NdbenchAWSCredentialProvider; +import com.netflix.ndbench.plugin.dynamodb.NdbenchAWSCredentialProviderV2; +import software.amazon.awssdk.auth.credentials.AwsCredentialsProvider; /** * @@ -32,6 +36,8 @@ public class DynamoDBModule extends AbstractModule { @Override protected void configure() { + bind(AWSCredentialsProvider.class).to(NdbenchAWSCredentialProvider.class); + bind(AwsCredentialsProvider.class).to(NdbenchAWSCredentialProviderV2.class); } @Provides diff --git a/ndbench-dynamodb-plugins/src/main/java/com/netflix/ndbench/plugin/dynamodb/operations/dynamodb/dataplane/CapacityConsumingFunction.java b/ndbench-dynamodb-plugins/src/main/java/com/netflix/ndbench/plugin/dynamodb/operations/CapacityConsumingFunction.java similarity index 61% rename from ndbench-dynamodb-plugins/src/main/java/com/netflix/ndbench/plugin/dynamodb/operations/dynamodb/dataplane/CapacityConsumingFunction.java rename to ndbench-dynamodb-plugins/src/main/java/com/netflix/ndbench/plugin/dynamodb/operations/CapacityConsumingFunction.java index 8a1da667..6610f614 100644 --- a/ndbench-dynamodb-plugins/src/main/java/com/netflix/ndbench/plugin/dynamodb/operations/dynamodb/dataplane/CapacityConsumingFunction.java +++ b/ndbench-dynamodb-plugins/src/main/java/com/netflix/ndbench/plugin/dynamodb/operations/CapacityConsumingFunction.java @@ -14,10 +14,19 @@ * limitations under the License. * */ -package com.netflix.ndbench.plugin.dynamodb.operations.dynamodb.dataplane; +package com.netflix.ndbench.plugin.dynamodb.operations; import java.util.function.Function; +/** + * This class will allow generalizes data plane operations between AWS SDK v1 and v2. + * @param This is the class of the result type from which to pull consumed capacity + * @param This is the class of input into the data plane operation + * @param This is the class of output of the data plane operation, after being interpreted from T + * + * @author Alexander Patrikalakis + */ public interface CapacityConsumingFunction extends Function { T measureConsumedCapacity(T t); + double getAndResetConsumed(); } diff --git a/ndbench-dynamodb-plugins/src/main/java/com/netflix/ndbench/plugin/dynamodb/operations/AbstractAwsOperation.java b/ndbench-dynamodb-plugins/src/main/java/com/netflix/ndbench/plugin/dynamodb/operations/v1/AbstractAwsOperation.java similarity index 97% rename from ndbench-dynamodb-plugins/src/main/java/com/netflix/ndbench/plugin/dynamodb/operations/AbstractAwsOperation.java rename to ndbench-dynamodb-plugins/src/main/java/com/netflix/ndbench/plugin/dynamodb/operations/v1/AbstractAwsOperation.java index 4551702a..5efc5be2 100644 --- a/ndbench-dynamodb-plugins/src/main/java/com/netflix/ndbench/plugin/dynamodb/operations/AbstractAwsOperation.java +++ b/ndbench-dynamodb-plugins/src/main/java/com/netflix/ndbench/plugin/dynamodb/operations/v1/AbstractAwsOperation.java @@ -14,7 +14,7 @@ * limitations under the License. * */ -package com.netflix.ndbench.plugin.dynamodb.operations; +package com.netflix.ndbench.plugin.dynamodb.operations.v1; import com.amazonaws.AmazonClientException; import com.amazonaws.AmazonServiceException; diff --git a/ndbench-dynamodb-plugins/src/main/java/com/netflix/ndbench/plugin/dynamodb/operations/cloudwatch/AbstractCloudWatchOperation.java b/ndbench-dynamodb-plugins/src/main/java/com/netflix/ndbench/plugin/dynamodb/operations/v1/cloudwatch/AbstractCloudWatchOperation.java similarity index 86% rename from ndbench-dynamodb-plugins/src/main/java/com/netflix/ndbench/plugin/dynamodb/operations/cloudwatch/AbstractCloudWatchOperation.java rename to ndbench-dynamodb-plugins/src/main/java/com/netflix/ndbench/plugin/dynamodb/operations/v1/cloudwatch/AbstractCloudWatchOperation.java index 93ccee72..0f2ebeea 100644 --- a/ndbench-dynamodb-plugins/src/main/java/com/netflix/ndbench/plugin/dynamodb/operations/cloudwatch/AbstractCloudWatchOperation.java +++ b/ndbench-dynamodb-plugins/src/main/java/com/netflix/ndbench/plugin/dynamodb/operations/v1/cloudwatch/AbstractCloudWatchOperation.java @@ -14,10 +14,10 @@ * limitations under the License. * */ -package com.netflix.ndbench.plugin.dynamodb.operations.cloudwatch; +package com.netflix.ndbench.plugin.dynamodb.operations.v1.cloudwatch; import com.amazonaws.services.cloudwatch.AmazonCloudWatch; -import com.netflix.ndbench.plugin.dynamodb.operations.AbstractAwsOperation; +import com.netflix.ndbench.plugin.dynamodb.operations.v1.AbstractAwsOperation; /** * @author Alexander Patrikalakis diff --git a/ndbench-dynamodb-plugins/src/main/java/com/netflix/ndbench/plugin/dynamodb/operations/cloudwatch/controlplane/PutMetricAlarmOperation.java b/ndbench-dynamodb-plugins/src/main/java/com/netflix/ndbench/plugin/dynamodb/operations/v1/cloudwatch/controlplane/PutMetricAlarmOperation.java similarity index 89% rename from ndbench-dynamodb-plugins/src/main/java/com/netflix/ndbench/plugin/dynamodb/operations/cloudwatch/controlplane/PutMetricAlarmOperation.java rename to ndbench-dynamodb-plugins/src/main/java/com/netflix/ndbench/plugin/dynamodb/operations/v1/cloudwatch/controlplane/PutMetricAlarmOperation.java index a85292b6..88bb24fd 100644 --- a/ndbench-dynamodb-plugins/src/main/java/com/netflix/ndbench/plugin/dynamodb/operations/cloudwatch/controlplane/PutMetricAlarmOperation.java +++ b/ndbench-dynamodb-plugins/src/main/java/com/netflix/ndbench/plugin/dynamodb/operations/v1/cloudwatch/controlplane/PutMetricAlarmOperation.java @@ -14,14 +14,14 @@ * limitations under the License. * */ -package com.netflix.ndbench.plugin.dynamodb.operations.cloudwatch.controlplane; +package com.netflix.ndbench.plugin.dynamodb.operations.v1.cloudwatch.controlplane; import com.amazonaws.AmazonClientException; import com.amazonaws.AmazonServiceException; import com.amazonaws.services.cloudwatch.AmazonCloudWatch; import com.amazonaws.services.cloudwatch.model.PutMetricAlarmRequest; import com.amazonaws.services.cloudwatch.model.PutMetricAlarmResult; -import com.netflix.ndbench.plugin.dynamodb.operations.cloudwatch.AbstractCloudWatchOperation; +import com.netflix.ndbench.plugin.dynamodb.operations.v1.cloudwatch.AbstractCloudWatchOperation; import java.util.function.Function; diff --git a/ndbench-dynamodb-plugins/src/main/java/com/netflix/ndbench/plugin/dynamodb/operations/cloudwatch/dataplane/PutMetricDataOperation.java b/ndbench-dynamodb-plugins/src/main/java/com/netflix/ndbench/plugin/dynamodb/operations/v1/cloudwatch/dataplane/PutMetricDataOperation.java similarity index 90% rename from ndbench-dynamodb-plugins/src/main/java/com/netflix/ndbench/plugin/dynamodb/operations/cloudwatch/dataplane/PutMetricDataOperation.java rename to ndbench-dynamodb-plugins/src/main/java/com/netflix/ndbench/plugin/dynamodb/operations/v1/cloudwatch/dataplane/PutMetricDataOperation.java index c427ccc6..0cb2c7c6 100644 --- a/ndbench-dynamodb-plugins/src/main/java/com/netflix/ndbench/plugin/dynamodb/operations/cloudwatch/dataplane/PutMetricDataOperation.java +++ b/ndbench-dynamodb-plugins/src/main/java/com/netflix/ndbench/plugin/dynamodb/operations/v1/cloudwatch/dataplane/PutMetricDataOperation.java @@ -14,14 +14,14 @@ * limitations under the License. * */ -package com.netflix.ndbench.plugin.dynamodb.operations.cloudwatch.dataplane; +package com.netflix.ndbench.plugin.dynamodb.operations.v1.cloudwatch.dataplane; import com.amazonaws.AmazonClientException; import com.amazonaws.AmazonServiceException; import com.amazonaws.services.cloudwatch.AmazonCloudWatch; import com.amazonaws.services.cloudwatch.model.PutMetricDataRequest; import com.amazonaws.services.cloudwatch.model.PutMetricDataResult; -import com.netflix.ndbench.plugin.dynamodb.operations.cloudwatch.AbstractCloudWatchOperation; +import com.netflix.ndbench.plugin.dynamodb.operations.v1.cloudwatch.AbstractCloudWatchOperation; import java.util.function.Function; diff --git a/ndbench-dynamodb-plugins/src/main/java/com/netflix/ndbench/plugin/dynamodb/operations/dynamodb/AbstractDynamoDBOperation.java b/ndbench-dynamodb-plugins/src/main/java/com/netflix/ndbench/plugin/dynamodb/operations/v1/dynamodb/AbstractDynamoDBOperation.java similarity index 92% rename from ndbench-dynamodb-plugins/src/main/java/com/netflix/ndbench/plugin/dynamodb/operations/dynamodb/AbstractDynamoDBOperation.java rename to ndbench-dynamodb-plugins/src/main/java/com/netflix/ndbench/plugin/dynamodb/operations/v1/dynamodb/AbstractDynamoDBOperation.java index 99b2458c..351bad90 100644 --- a/ndbench-dynamodb-plugins/src/main/java/com/netflix/ndbench/plugin/dynamodb/operations/dynamodb/AbstractDynamoDBOperation.java +++ b/ndbench-dynamodb-plugins/src/main/java/com/netflix/ndbench/plugin/dynamodb/operations/v1/dynamodb/AbstractDynamoDBOperation.java @@ -14,11 +14,11 @@ * limitations under the License. * */ -package com.netflix.ndbench.plugin.dynamodb.operations.dynamodb; +package com.netflix.ndbench.plugin.dynamodb.operations.v1.dynamodb; import com.amazonaws.services.dynamodbv2.AmazonDynamoDB; import com.google.common.base.Preconditions; -import com.netflix.ndbench.plugin.dynamodb.operations.AbstractAwsOperation; +import com.netflix.ndbench.plugin.dynamodb.operations.v1.AbstractAwsOperation; import org.apache.commons.lang3.StringUtils; diff --git a/ndbench-dynamodb-plugins/src/main/java/com/netflix/ndbench/plugin/dynamodb/operations/dynamodb/controlplane/CreateDynamoDBTable.java b/ndbench-dynamodb-plugins/src/main/java/com/netflix/ndbench/plugin/dynamodb/operations/v1/dynamodb/controlplane/CreateDynamoDBTable.java similarity index 96% rename from ndbench-dynamodb-plugins/src/main/java/com/netflix/ndbench/plugin/dynamodb/operations/dynamodb/controlplane/CreateDynamoDBTable.java rename to ndbench-dynamodb-plugins/src/main/java/com/netflix/ndbench/plugin/dynamodb/operations/v1/dynamodb/controlplane/CreateDynamoDBTable.java index 91103635..8e77ea70 100644 --- a/ndbench-dynamodb-plugins/src/main/java/com/netflix/ndbench/plugin/dynamodb/operations/dynamodb/controlplane/CreateDynamoDBTable.java +++ b/ndbench-dynamodb-plugins/src/main/java/com/netflix/ndbench/plugin/dynamodb/operations/v1/dynamodb/controlplane/CreateDynamoDBTable.java @@ -14,7 +14,7 @@ * limitations under the License. * */ -package com.netflix.ndbench.plugin.dynamodb.operations.dynamodb.controlplane; +package com.netflix.ndbench.plugin.dynamodb.operations.v1.dynamodb.controlplane; import com.amazonaws.services.dynamodbv2.AmazonDynamoDB; import com.amazonaws.services.dynamodbv2.model.AttributeDefinition; @@ -27,7 +27,7 @@ import com.amazonaws.services.dynamodbv2.model.TableDescription; import com.amazonaws.services.dynamodbv2.util.TableUtils; import com.google.common.base.Preconditions; -import com.netflix.ndbench.plugin.dynamodb.operations.dynamodb.AbstractDynamoDBOperation; +import com.netflix.ndbench.plugin.dynamodb.operations.v1.dynamodb.AbstractDynamoDBOperation; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/ndbench-dynamodb-plugins/src/main/java/com/netflix/ndbench/plugin/dynamodb/operations/dynamodb/controlplane/DeleteDynamoDBTable.java b/ndbench-dynamodb-plugins/src/main/java/com/netflix/ndbench/plugin/dynamodb/operations/v1/dynamodb/controlplane/DeleteDynamoDBTable.java similarity index 91% rename from ndbench-dynamodb-plugins/src/main/java/com/netflix/ndbench/plugin/dynamodb/operations/dynamodb/controlplane/DeleteDynamoDBTable.java rename to ndbench-dynamodb-plugins/src/main/java/com/netflix/ndbench/plugin/dynamodb/operations/v1/dynamodb/controlplane/DeleteDynamoDBTable.java index 2fe2847b..0e1d216a 100644 --- a/ndbench-dynamodb-plugins/src/main/java/com/netflix/ndbench/plugin/dynamodb/operations/dynamodb/controlplane/DeleteDynamoDBTable.java +++ b/ndbench-dynamodb-plugins/src/main/java/com/netflix/ndbench/plugin/dynamodb/operations/v1/dynamodb/controlplane/DeleteDynamoDBTable.java @@ -14,12 +14,12 @@ * limitations under the License. * */ -package com.netflix.ndbench.plugin.dynamodb.operations.dynamodb.controlplane; +package com.netflix.ndbench.plugin.dynamodb.operations.v1.dynamodb.controlplane; import com.amazonaws.services.dynamodbv2.AmazonDynamoDB; import com.amazonaws.services.dynamodbv2.document.Table; import com.amazonaws.services.dynamodbv2.model.ResourceNotFoundException; -import com.netflix.ndbench.plugin.dynamodb.operations.dynamodb.AbstractDynamoDBOperation; +import com.netflix.ndbench.plugin.dynamodb.operations.v1.dynamodb.AbstractDynamoDBOperation; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/ndbench-dynamodb-plugins/src/main/java/com/netflix/ndbench/plugin/dynamodb/operations/dynamodb/controlplane/DescribeLimits.java b/ndbench-dynamodb-plugins/src/main/java/com/netflix/ndbench/plugin/dynamodb/operations/v1/dynamodb/controlplane/DescribeLimits.java similarity index 88% rename from ndbench-dynamodb-plugins/src/main/java/com/netflix/ndbench/plugin/dynamodb/operations/dynamodb/controlplane/DescribeLimits.java rename to ndbench-dynamodb-plugins/src/main/java/com/netflix/ndbench/plugin/dynamodb/operations/v1/dynamodb/controlplane/DescribeLimits.java index eb233dac..fbfe1080 100644 --- a/ndbench-dynamodb-plugins/src/main/java/com/netflix/ndbench/plugin/dynamodb/operations/dynamodb/controlplane/DescribeLimits.java +++ b/ndbench-dynamodb-plugins/src/main/java/com/netflix/ndbench/plugin/dynamodb/operations/v1/dynamodb/controlplane/DescribeLimits.java @@ -14,12 +14,12 @@ * limitations under the License. * */ -package com.netflix.ndbench.plugin.dynamodb.operations.dynamodb.controlplane; +package com.netflix.ndbench.plugin.dynamodb.operations.v1.dynamodb.controlplane; import com.amazonaws.services.dynamodbv2.AmazonDynamoDB; import com.amazonaws.services.dynamodbv2.model.DescribeLimitsRequest; import com.amazonaws.services.dynamodbv2.model.DescribeLimitsResult; -import com.netflix.ndbench.plugin.dynamodb.operations.dynamodb.AbstractDynamoDBOperation; +import com.netflix.ndbench.plugin.dynamodb.operations.v1.dynamodb.AbstractDynamoDBOperation; import java.util.function.Supplier; diff --git a/ndbench-dynamodb-plugins/src/main/java/com/netflix/ndbench/plugin/dynamodb/operations/dynamodb/dataplane/AbstractDynamoDBDataPlaneOperation.java b/ndbench-dynamodb-plugins/src/main/java/com/netflix/ndbench/plugin/dynamodb/operations/v1/dynamodb/dataplane/AbstractDynamoDBDataPlaneOperation.java similarity index 83% rename from ndbench-dynamodb-plugins/src/main/java/com/netflix/ndbench/plugin/dynamodb/operations/dynamodb/dataplane/AbstractDynamoDBDataPlaneOperation.java rename to ndbench-dynamodb-plugins/src/main/java/com/netflix/ndbench/plugin/dynamodb/operations/v1/dynamodb/dataplane/AbstractDynamoDBDataPlaneOperation.java index 47e44385..51f9d9c9 100644 --- a/ndbench-dynamodb-plugins/src/main/java/com/netflix/ndbench/plugin/dynamodb/operations/dynamodb/dataplane/AbstractDynamoDBDataPlaneOperation.java +++ b/ndbench-dynamodb-plugins/src/main/java/com/netflix/ndbench/plugin/dynamodb/operations/v1/dynamodb/dataplane/AbstractDynamoDBDataPlaneOperation.java @@ -14,7 +14,7 @@ * limitations under the License. * */ -package com.netflix.ndbench.plugin.dynamodb.operations.dynamodb.dataplane; +package com.netflix.ndbench.plugin.dynamodb.operations.v1.dynamodb.dataplane; import com.amazonaws.services.dynamodbv2.AmazonDynamoDB; import com.amazonaws.services.dynamodbv2.model.ConsumedCapacity; @@ -22,14 +22,15 @@ import com.google.common.base.Preconditions; import com.google.common.util.concurrent.AtomicDouble; import com.netflix.ndbench.api.plugin.DataGenerator; -import com.netflix.ndbench.plugin.dynamodb.operations.dynamodb.AbstractDynamoDBOperation; +import com.netflix.ndbench.plugin.dynamodb.operations.CapacityConsumingFunction; +import com.netflix.ndbench.plugin.dynamodb.operations.v1.dynamodb.AbstractDynamoDBOperation; import java.util.List; /** * @author Alexander Patrikalakis */ -public class AbstractDynamoDBDataPlaneOperation extends AbstractDynamoDBOperation { +public abstract class AbstractDynamoDBDataPlaneOperation extends AbstractDynamoDBOperation implements CapacityConsumingFunction { protected final DataGenerator dataGenerator; protected final AtomicDouble consumed = new AtomicDouble(0.0); protected final ReturnConsumedCapacity returnConsumedCapacity; @@ -51,6 +52,7 @@ protected double getConsumedCapacityForTable(List consumedCapa .orElse(0.0); } + @Override public double getAndResetConsumed() { return consumed.getAndSet(0.0); } diff --git a/ndbench-dynamodb-plugins/src/main/java/com/netflix/ndbench/plugin/dynamodb/operations/dynamodb/dataplane/AbstractDynamoDBReadOperation.java b/ndbench-dynamodb-plugins/src/main/java/com/netflix/ndbench/plugin/dynamodb/operations/v1/dynamodb/dataplane/AbstractDynamoDBReadOperation.java similarity index 87% rename from ndbench-dynamodb-plugins/src/main/java/com/netflix/ndbench/plugin/dynamodb/operations/dynamodb/dataplane/AbstractDynamoDBReadOperation.java rename to ndbench-dynamodb-plugins/src/main/java/com/netflix/ndbench/plugin/dynamodb/operations/v1/dynamodb/dataplane/AbstractDynamoDBReadOperation.java index 33aa9be5..2f0da480 100644 --- a/ndbench-dynamodb-plugins/src/main/java/com/netflix/ndbench/plugin/dynamodb/operations/dynamodb/dataplane/AbstractDynamoDBReadOperation.java +++ b/ndbench-dynamodb-plugins/src/main/java/com/netflix/ndbench/plugin/dynamodb/operations/v1/dynamodb/dataplane/AbstractDynamoDBReadOperation.java @@ -14,7 +14,7 @@ * limitations under the License. * */ -package com.netflix.ndbench.plugin.dynamodb.operations.dynamodb.dataplane; +package com.netflix.ndbench.plugin.dynamodb.operations.v1.dynamodb.dataplane; import com.amazonaws.services.dynamodbv2.AmazonDynamoDB; import com.amazonaws.services.dynamodbv2.model.ReturnConsumedCapacity; @@ -23,7 +23,7 @@ /** * @author Alexander Patrikalakis */ -public abstract class AbstractDynamoDBReadOperation extends AbstractDynamoDBDataPlaneOperation { +public abstract class AbstractDynamoDBReadOperation extends AbstractDynamoDBDataPlaneOperation { protected final boolean consistentRead; protected AbstractDynamoDBReadOperation(DataGenerator dataGenerator, AmazonDynamoDB dynamoDB, String tableName, diff --git a/ndbench-dynamodb-plugins/src/main/java/com/netflix/ndbench/plugin/dynamodb/operations/dynamodb/dataplane/DynamoDBReadBulk.java b/ndbench-dynamodb-plugins/src/main/java/com/netflix/ndbench/plugin/dynamodb/operations/v1/dynamodb/dataplane/DynamoDBReadBulk.java similarity index 95% rename from ndbench-dynamodb-plugins/src/main/java/com/netflix/ndbench/plugin/dynamodb/operations/dynamodb/dataplane/DynamoDBReadBulk.java rename to ndbench-dynamodb-plugins/src/main/java/com/netflix/ndbench/plugin/dynamodb/operations/v1/dynamodb/dataplane/DynamoDBReadBulk.java index e9fdabb6..be49f167 100644 --- a/ndbench-dynamodb-plugins/src/main/java/com/netflix/ndbench/plugin/dynamodb/operations/dynamodb/dataplane/DynamoDBReadBulk.java +++ b/ndbench-dynamodb-plugins/src/main/java/com/netflix/ndbench/plugin/dynamodb/operations/v1/dynamodb/dataplane/DynamoDBReadBulk.java @@ -14,7 +14,7 @@ * limitations under the License. * */ -package com.netflix.ndbench.plugin.dynamodb.operations.dynamodb.dataplane; +package com.netflix.ndbench.plugin.dynamodb.operations.v1.dynamodb.dataplane; import com.amazonaws.AmazonClientException; import com.amazonaws.AmazonServiceException; @@ -37,8 +37,7 @@ * @author Alexander Patrikalakis * @author ipapapa */ -public class DynamoDBReadBulk extends AbstractDynamoDBReadOperation - implements CapacityConsumingFunction, List> { +public class DynamoDBReadBulk extends AbstractDynamoDBReadOperation, List> { public DynamoDBReadBulk(DataGenerator dataGenerator, AmazonDynamoDB dynamoDB, String tableName, String partitionKeyName, boolean consistentRead, ReturnConsumedCapacity returnConsumedCapacity) { @@ -61,6 +60,12 @@ public List apply(List keys) { } } + @Override + public BatchGetItemResult measureConsumedCapacity(BatchGetItemResult result) { + consumed.addAndGet(result.getConsumedCapacity() == null ? 0 : getConsumedCapacityForTable(result.getConsumedCapacity())); + return result; + } + private KeysAndAttributes generateReadRequests(List keys) { return new KeysAndAttributes() .withKeys(keys.stream() @@ -85,10 +90,4 @@ private BatchGetItemResult runBatchGetRequest(KeysAndAttributes keysAndAttribute .withRequestItems(ImmutableMap.of(tableName, keysAndAttributes)) .withReturnConsumedCapacity(returnConsumedCapacity))); } - - @Override - public BatchGetItemResult measureConsumedCapacity(BatchGetItemResult result) { - consumed.addAndGet(result.getConsumedCapacity() == null ? 0 : getConsumedCapacityForTable(result.getConsumedCapacity())); - return result; - } } diff --git a/ndbench-dynamodb-plugins/src/main/java/com/netflix/ndbench/plugin/dynamodb/operations/dynamodb/dataplane/DynamoDBReadSingle.java b/ndbench-dynamodb-plugins/src/main/java/com/netflix/ndbench/plugin/dynamodb/operations/v1/dynamodb/dataplane/DynamoDBReadSingle.java similarity index 94% rename from ndbench-dynamodb-plugins/src/main/java/com/netflix/ndbench/plugin/dynamodb/operations/dynamodb/dataplane/DynamoDBReadSingle.java rename to ndbench-dynamodb-plugins/src/main/java/com/netflix/ndbench/plugin/dynamodb/operations/v1/dynamodb/dataplane/DynamoDBReadSingle.java index b1ff3812..26e151ca 100644 --- a/ndbench-dynamodb-plugins/src/main/java/com/netflix/ndbench/plugin/dynamodb/operations/dynamodb/dataplane/DynamoDBReadSingle.java +++ b/ndbench-dynamodb-plugins/src/main/java/com/netflix/ndbench/plugin/dynamodb/operations/v1/dynamodb/dataplane/DynamoDBReadSingle.java @@ -14,7 +14,7 @@ * limitations under the License. * */ -package com.netflix.ndbench.plugin.dynamodb.operations.dynamodb.dataplane; +package com.netflix.ndbench.plugin.dynamodb.operations.v1.dynamodb.dataplane; import com.amazonaws.AmazonClientException; import com.amazonaws.AmazonServiceException; @@ -34,8 +34,7 @@ * @author Alexander Patrikalakis * @author ipapapa */ -public class DynamoDBReadSingle extends AbstractDynamoDBReadOperation - implements CapacityConsumingFunction { +public class DynamoDBReadSingle extends AbstractDynamoDBReadOperation { public DynamoDBReadSingle(DataGenerator dataGenerator, AmazonDynamoDB dynamoDB, String tableName, String partitionKeyName, boolean consistentRead, ReturnConsumedCapacity returnConsumedCapacity) { diff --git a/ndbench-dynamodb-plugins/src/main/java/com/netflix/ndbench/plugin/dynamodb/operations/dynamodb/dataplane/DynamoDBWriteBulk.java b/ndbench-dynamodb-plugins/src/main/java/com/netflix/ndbench/plugin/dynamodb/operations/v1/dynamodb/dataplane/DynamoDBWriteBulk.java similarity index 95% rename from ndbench-dynamodb-plugins/src/main/java/com/netflix/ndbench/plugin/dynamodb/operations/dynamodb/dataplane/DynamoDBWriteBulk.java rename to ndbench-dynamodb-plugins/src/main/java/com/netflix/ndbench/plugin/dynamodb/operations/v1/dynamodb/dataplane/DynamoDBWriteBulk.java index baf42eac..7f428ca7 100644 --- a/ndbench-dynamodb-plugins/src/main/java/com/netflix/ndbench/plugin/dynamodb/operations/dynamodb/dataplane/DynamoDBWriteBulk.java +++ b/ndbench-dynamodb-plugins/src/main/java/com/netflix/ndbench/plugin/dynamodb/operations/v1/dynamodb/dataplane/DynamoDBWriteBulk.java @@ -14,7 +14,7 @@ * limitations under the License. * */ -package com.netflix.ndbench.plugin.dynamodb.operations.dynamodb.dataplane; +package com.netflix.ndbench.plugin.dynamodb.operations.v1.dynamodb.dataplane; import com.amazonaws.AmazonClientException; import com.amazonaws.AmazonServiceException; @@ -37,8 +37,7 @@ * @author Alexander Patrikalakis * @author ipapapa */ -public class DynamoDBWriteBulk extends AbstractDynamoDBDataPlaneOperation - implements CapacityConsumingFunction, List> { +public class DynamoDBWriteBulk extends AbstractDynamoDBDataPlaneOperation, List> { public DynamoDBWriteBulk(DataGenerator dataGenerator, AmazonDynamoDB dynamoDB, String tableName, String partitionKeyName, ReturnConsumedCapacity returnConsumedCapacity) { super(dynamoDB, tableName, partitionKeyName, dataGenerator, returnConsumedCapacity); @@ -61,6 +60,12 @@ public List apply(List keys) { } } + @Override + public BatchWriteItemResult measureConsumedCapacity(BatchWriteItemResult result) { + consumed.addAndGet(result.getConsumedCapacity() == null ? 0 : getConsumedCapacityForTable(result.getConsumedCapacity())); + return result; + } + private List generateWriteRequests(List keys) { return keys.stream() .map(key -> ImmutableMap.of(partitionKeyName, new AttributeValue(key), @@ -85,10 +90,4 @@ private BatchWriteItemResult runBatchWriteRequest(List writeReques .withRequestItems(ImmutableMap.of(tableName, writeRequests)) .withReturnConsumedCapacity(returnConsumedCapacity))); } - - @Override - public BatchWriteItemResult measureConsumedCapacity(BatchWriteItemResult result) { - consumed.addAndGet(result.getConsumedCapacity() == null ? 0 : getConsumedCapacityForTable(result.getConsumedCapacity())); - return result; - } } diff --git a/ndbench-dynamodb-plugins/src/main/java/com/netflix/ndbench/plugin/dynamodb/operations/dynamodb/dataplane/DynamoDBWriteSingle.java b/ndbench-dynamodb-plugins/src/main/java/com/netflix/ndbench/plugin/dynamodb/operations/v1/dynamodb/dataplane/DynamoDBWriteSingle.java similarity index 94% rename from ndbench-dynamodb-plugins/src/main/java/com/netflix/ndbench/plugin/dynamodb/operations/dynamodb/dataplane/DynamoDBWriteSingle.java rename to ndbench-dynamodb-plugins/src/main/java/com/netflix/ndbench/plugin/dynamodb/operations/v1/dynamodb/dataplane/DynamoDBWriteSingle.java index 1c1665f1..65346b2c 100644 --- a/ndbench-dynamodb-plugins/src/main/java/com/netflix/ndbench/plugin/dynamodb/operations/dynamodb/dataplane/DynamoDBWriteSingle.java +++ b/ndbench-dynamodb-plugins/src/main/java/com/netflix/ndbench/plugin/dynamodb/operations/v1/dynamodb/dataplane/DynamoDBWriteSingle.java @@ -14,7 +14,7 @@ * limitations under the License. * */ -package com.netflix.ndbench.plugin.dynamodb.operations.dynamodb.dataplane; +package com.netflix.ndbench.plugin.dynamodb.operations.v1.dynamodb.dataplane; import com.amazonaws.AmazonClientException; import com.amazonaws.AmazonServiceException; @@ -32,8 +32,7 @@ * @author Alexander Patrikalakis * @author ipapapa */ -public class DynamoDBWriteSingle extends AbstractDynamoDBDataPlaneOperation - implements CapacityConsumingFunction { +public class DynamoDBWriteSingle extends AbstractDynamoDBDataPlaneOperation { public DynamoDBWriteSingle(DataGenerator dataGenerator, AmazonDynamoDB dynamoDB, String tableName, String partitionKeyName, ReturnConsumedCapacity returnConsumedCapacity) { super(dynamoDB, tableName, partitionKeyName, dataGenerator, returnConsumedCapacity); diff --git a/ndbench-dynamodb-plugins/src/main/java/com/netflix/ndbench/plugin/dynamodb/operations/v2/AbstractAwsOperation.java b/ndbench-dynamodb-plugins/src/main/java/com/netflix/ndbench/plugin/dynamodb/operations/v2/AbstractAwsOperation.java new file mode 100644 index 00000000..aa99d84a --- /dev/null +++ b/ndbench-dynamodb-plugins/src/main/java/com/netflix/ndbench/plugin/dynamodb/operations/v2/AbstractAwsOperation.java @@ -0,0 +1,44 @@ +/* + * Copyright 2018 Netflix, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ +package com.netflix.ndbench.plugin.dynamodb.operations.v2; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import software.amazon.awssdk.awscore.exception.AwsServiceException; +import software.amazon.awssdk.core.exception.SdkClientException; + +public abstract class AbstractAwsOperation { + private static final Logger logger = LoggerFactory.getLogger(AbstractAwsOperation.class); + + protected AwsServiceException logAwsServiceException(AwsServiceException ase) { + logger.error("Caught an AmazonServiceException, which means your request made it " + + "to AWS, but was rejected with an error response for some reason."); + logger.error("Error Message: " + ase.getMessage()); + logger.error("HTTP Status Code: " + ase.statusCode()); + logger.error("AWS Error Code: " + ase.awsErrorDetails().errorCode()); + logger.error("Request ID: " + ase.requestId()); + return ase; + } + + protected SdkClientException logSdkClientException(SdkClientException ace) { + logger.error("Caught an AmazonClientException, which means the client encountered " + + "a serious internal problem while trying to communicate with AWS, " + + "such as not being able to access the network."); + logger.error("Error Message: " + ace.getMessage()); + return ace; + } +} diff --git a/ndbench-dynamodb-plugins/src/main/java/com/netflix/ndbench/plugin/dynamodb/operations/v2/dynamodb/AbstractDynamoDBOperation.java b/ndbench-dynamodb-plugins/src/main/java/com/netflix/ndbench/plugin/dynamodb/operations/v2/dynamodb/AbstractDynamoDBOperation.java new file mode 100644 index 00000000..49d0bd4b --- /dev/null +++ b/ndbench-dynamodb-plugins/src/main/java/com/netflix/ndbench/plugin/dynamodb/operations/v2/dynamodb/AbstractDynamoDBOperation.java @@ -0,0 +1,69 @@ +/* + * Copyright 2018 Netflix, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ +package com.netflix.ndbench.plugin.dynamodb.operations.v2.dynamodb; + +import com.google.common.base.Preconditions; +import com.netflix.ndbench.plugin.dynamodb.operations.v2.AbstractAwsOperation; +import org.apache.commons.lang3.StringUtils; +import software.amazon.awssdk.awscore.exception.AwsServiceException; +import software.amazon.awssdk.core.exception.SdkClientException; +import software.amazon.awssdk.services.dynamodb.DynamoDbAsyncClient; + +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.ExecutionException; + +/** + * @author Alexander Patrikalakis + * @author ipapapa + */ +public abstract class AbstractDynamoDBOperation extends AbstractAwsOperation { + protected static final String ATTRIBUTE_NAME = "value"; + + protected final DynamoDbAsyncClient dynamoDB; + protected final String tableName; + protected final String partitionKeyName; + private final String operationName; + + protected AbstractDynamoDBOperation(DynamoDbAsyncClient dynamoDB, + String tableName, + String partitionKeyName, String operationName) { + Preconditions.checkNotNull(dynamoDB, "DynamoDB client must not be null"); + Preconditions.checkArgument(StringUtils.isNotEmpty(tableName), "Invalid table name"); + Preconditions.checkArgument(StringUtils.isNotEmpty(partitionKeyName), "Invalid partition key name"); + this.dynamoDB = dynamoDB; + this.tableName = tableName; + this.partitionKeyName = partitionKeyName; + this.operationName = operationName; + } + + public R completeAndCatch(CompletableFuture future) { + try { + return future.get(); + } catch (InterruptedException e) { + throw new RuntimeException(operationName + " request was interrupted.", e); + } catch (ExecutionException e) { + final Throwable cause = e.getCause(); + if (cause instanceof AwsServiceException) { + throw logAwsServiceException((AwsServiceException) cause); + } else if (cause instanceof SdkClientException) { + throw logSdkClientException((SdkClientException) cause); + } else { + throw new RuntimeException(operationName + " request failed.", cause); + } + } + } +} diff --git a/ndbench-dynamodb-plugins/src/main/java/com/netflix/ndbench/plugin/dynamodb/operations/v2/dynamodb/controlplane/CreateDynamoDBTable.java b/ndbench-dynamodb-plugins/src/main/java/com/netflix/ndbench/plugin/dynamodb/operations/v2/dynamodb/controlplane/CreateDynamoDBTable.java new file mode 100644 index 00000000..930ca0cb --- /dev/null +++ b/ndbench-dynamodb-plugins/src/main/java/com/netflix/ndbench/plugin/dynamodb/operations/v2/dynamodb/controlplane/CreateDynamoDBTable.java @@ -0,0 +1,99 @@ +/* + * Copyright 2018 Netflix, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ +package com.netflix.ndbench.plugin.dynamodb.operations.v2.dynamodb.controlplane; + +import com.google.common.base.Preconditions; +import com.netflix.ndbench.plugin.dynamodb.operations.v2.dynamodb.AbstractDynamoDBOperation; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import software.amazon.awssdk.services.dynamodb.DynamoDbAsyncClient; +import software.amazon.awssdk.services.dynamodb.model.AttributeDefinition; +import software.amazon.awssdk.services.dynamodb.model.CreateTableRequest; +import software.amazon.awssdk.services.dynamodb.model.DescribeTableRequest; +import software.amazon.awssdk.services.dynamodb.model.DescribeTableResponse; +import software.amazon.awssdk.services.dynamodb.model.KeySchemaElement; +import software.amazon.awssdk.services.dynamodb.model.KeyType; +import software.amazon.awssdk.services.dynamodb.model.ProvisionedThroughput; +import software.amazon.awssdk.services.dynamodb.model.ResourceInUseException; +import software.amazon.awssdk.services.dynamodb.model.ResourceNotFoundException; +import software.amazon.awssdk.services.dynamodb.model.ScalarAttributeType; +import software.amazon.awssdk.services.dynamodb.model.TableDescription; + +import java.util.ArrayList; +import java.util.function.Supplier; + +/** + * @author ipapapa + * @author Alexander Patrikalakis + */ +public class CreateDynamoDBTable extends AbstractDynamoDBOperation implements Supplier { + private static final Logger logger = LoggerFactory.getLogger(CreateDynamoDBTable.class); + private final long readCapacityUnits; + private final long writeCapacityUnits; + public CreateDynamoDBTable(DynamoDbAsyncClient dynamoDB, String tableName, String partitionKeyName, + long readCapacityUnits, long writeCapacityUnits) { + super(dynamoDB, tableName, partitionKeyName, "CreateTable"); + Preconditions.checkArgument(readCapacityUnits > 0); + Preconditions.checkArgument(writeCapacityUnits > 0); + this.readCapacityUnits = readCapacityUnits; + this.writeCapacityUnits = writeCapacityUnits; + } + + @Override + public TableDescription get() { + /* + * Create a table with a primary hash key named 'name', which holds a string. + * Several properties such as provisioned throughput and atribute names are + * defined in the configuration interface. + */ + final DescribeTableResponse describeTableResponse; + try { + describeTableResponse + = completeAndCatch(dynamoDB.describeTable(DescribeTableRequest.builder().tableName(tableName).build())); + logger.info("Not creating table because it exists already"); + return describeTableResponse.table(); + } catch(ResourceNotFoundException e) { + logger.info("Creating Table: " + tableName); + } + + // key schema + ArrayList keySchema = new ArrayList<>(); + keySchema.add(KeySchemaElement.builder().attributeName(partitionKeyName).keyType(KeyType.HASH).build()); + + // Attribute definitions + ArrayList attributeDefinitions = new ArrayList<>(); + attributeDefinitions.add(AttributeDefinition.builder().attributeName(partitionKeyName) + .attributeType(ScalarAttributeType.S).build()); + /* + * constructing the table request: Schema + Attributed definitions + Provisioned + * throughput + */ + CreateTableRequest request = CreateTableRequest.builder().tableName(tableName) + .keySchema(keySchema).attributeDefinitions(attributeDefinitions) + .provisionedThroughput(ProvisionedThroughput.builder().readCapacityUnits(readCapacityUnits) + .writeCapacityUnits(writeCapacityUnits).build()).build(); + + // Creating table + try { + completeAndCatch(dynamoDB.createTable(request)); + } catch(ResourceInUseException e) { + logger.info("Table already exists."); + } + + return completeAndCatch(dynamoDB.describeTable(DescribeTableRequest.builder().tableName(tableName).build())).table(); + } +} diff --git a/ndbench-dynamodb-plugins/src/main/java/com/netflix/ndbench/plugin/dynamodb/operations/v2/dynamodb/controlplane/DeleteDynamoDBTable.java b/ndbench-dynamodb-plugins/src/main/java/com/netflix/ndbench/plugin/dynamodb/operations/v2/dynamodb/controlplane/DeleteDynamoDBTable.java new file mode 100644 index 00000000..aee259d8 --- /dev/null +++ b/ndbench-dynamodb-plugins/src/main/java/com/netflix/ndbench/plugin/dynamodb/operations/v2/dynamodb/controlplane/DeleteDynamoDBTable.java @@ -0,0 +1,45 @@ +/* + * Copyright 2018 Netflix, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ +package com.netflix.ndbench.plugin.dynamodb.operations.v2.dynamodb.controlplane; + +import com.netflix.ndbench.plugin.dynamodb.operations.v2.dynamodb.AbstractDynamoDBOperation; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import software.amazon.awssdk.services.dynamodb.DynamoDbAsyncClient; +import software.amazon.awssdk.services.dynamodb.model.DeleteTableRequest; +import software.amazon.awssdk.services.dynamodb.model.ResourceNotFoundException; + +/** + * @author ipapapa + * @author Alexander Patrikalakis + */ +public class DeleteDynamoDBTable extends AbstractDynamoDBOperation { + private static final Logger logger = LoggerFactory.getLogger(DeleteDynamoDBTable.class); + + public DeleteDynamoDBTable(DynamoDbAsyncClient dynamoDB, String tableName, String partitionKeyName) { + super(dynamoDB, tableName, partitionKeyName, "DeleteTable"); + } + + public void delete() { + logger.info("Issuing DeleteTable request for " + tableName); + try { + completeAndCatch(dynamoDB.deleteTable(DeleteTableRequest.builder().tableName(tableName).build())); + } catch (ResourceNotFoundException e) { + logger.warn("Table is already deleted", e); + } + } +} diff --git a/ndbench-dynamodb-plugins/src/main/java/com/netflix/ndbench/plugin/dynamodb/operations/v2/dynamodb/controlplane/DescribeLimits.java b/ndbench-dynamodb-plugins/src/main/java/com/netflix/ndbench/plugin/dynamodb/operations/v2/dynamodb/controlplane/DescribeLimits.java new file mode 100644 index 00000000..71033c68 --- /dev/null +++ b/ndbench-dynamodb-plugins/src/main/java/com/netflix/ndbench/plugin/dynamodb/operations/v2/dynamodb/controlplane/DescribeLimits.java @@ -0,0 +1,37 @@ +/* + * Copyright 2018 Netflix, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ +package com.netflix.ndbench.plugin.dynamodb.operations.v2.dynamodb.controlplane; + +import com.netflix.ndbench.plugin.dynamodb.operations.v2.dynamodb.AbstractDynamoDBOperation; +import software.amazon.awssdk.services.dynamodb.DynamoDbAsyncClient; +import software.amazon.awssdk.services.dynamodb.model.DescribeLimitsResponse; + +import java.util.function.Supplier; + +/** + * @author Alexander Patrikalakis + */ +public class DescribeLimits extends AbstractDynamoDBOperation implements Supplier { + public DescribeLimits(DynamoDbAsyncClient dynamoDB, String tableName, String partitionKeyName) { + super(dynamoDB, tableName, partitionKeyName, "DescribeLimits"); + } + + @Override + public DescribeLimitsResponse get() { + return completeAndCatch(dynamoDB.describeLimits()); + } +} diff --git a/ndbench-dynamodb-plugins/src/main/java/com/netflix/ndbench/plugin/dynamodb/operations/v2/dynamodb/dataplane/AbstractDynamoDBDataPlaneOperation.java b/ndbench-dynamodb-plugins/src/main/java/com/netflix/ndbench/plugin/dynamodb/operations/v2/dynamodb/dataplane/AbstractDynamoDBDataPlaneOperation.java new file mode 100644 index 00000000..c36c0700 --- /dev/null +++ b/ndbench-dynamodb-plugins/src/main/java/com/netflix/ndbench/plugin/dynamodb/operations/v2/dynamodb/dataplane/AbstractDynamoDBDataPlaneOperation.java @@ -0,0 +1,59 @@ +/* + * Copyright 2018 Netflix, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ +package com.netflix.ndbench.plugin.dynamodb.operations.v2.dynamodb.dataplane; + +import com.google.common.util.concurrent.AtomicDouble; +import com.netflix.ndbench.api.plugin.DataGenerator; +import com.netflix.ndbench.plugin.dynamodb.operations.CapacityConsumingFunction; +import com.netflix.ndbench.plugin.dynamodb.operations.v2.dynamodb.AbstractDynamoDBOperation; + +import java.util.List; + +import software.amazon.awssdk.services.dynamodb.DynamoDbAsyncClient; +import software.amazon.awssdk.services.dynamodb.model.ConsumedCapacity; +import software.amazon.awssdk.services.dynamodb.model.ReturnConsumedCapacity; + +/** + * @author Alexander Patrikalakis + */ +public abstract class AbstractDynamoDBDataPlaneOperation extends AbstractDynamoDBOperation + implements CapacityConsumingFunction { + protected final DataGenerator dataGenerator; + protected final AtomicDouble consumed = new AtomicDouble(0.0); + protected final ReturnConsumedCapacity returnConsumedCapacity; + + protected AbstractDynamoDBDataPlaneOperation(DynamoDbAsyncClient dynamoDB, String tableName, String partitionKeyName, + DataGenerator dataGenerator, ReturnConsumedCapacity returnConsumedCapacity, + String operationName) { + super(dynamoDB, tableName, partitionKeyName, operationName); + this.dataGenerator = dataGenerator; + this.returnConsumedCapacity = returnConsumedCapacity; + } + + protected double getConsumedCapacityForTable(List consumedCapacities) { + return consumedCapacities.stream() + .filter(c -> tableName.equals(c.tableName())) + .map(ConsumedCapacity::capacityUnits) + .findFirst() + .orElse(0.0); + } + + @Override + public double getAndResetConsumed() { + return consumed.getAndSet(0.0); + } +} \ No newline at end of file diff --git a/ndbench-dynamodb-plugins/src/main/java/com/netflix/ndbench/plugin/dynamodb/operations/v2/dynamodb/dataplane/AbstractDynamoDBReadOperation.java b/ndbench-dynamodb-plugins/src/main/java/com/netflix/ndbench/plugin/dynamodb/operations/v2/dynamodb/dataplane/AbstractDynamoDBReadOperation.java new file mode 100644 index 00000000..2b8d8649 --- /dev/null +++ b/ndbench-dynamodb-plugins/src/main/java/com/netflix/ndbench/plugin/dynamodb/operations/v2/dynamodb/dataplane/AbstractDynamoDBReadOperation.java @@ -0,0 +1,35 @@ +/* + * Copyright 2018 Netflix, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ +package com.netflix.ndbench.plugin.dynamodb.operations.v2.dynamodb.dataplane; + +import com.netflix.ndbench.api.plugin.DataGenerator; +import software.amazon.awssdk.services.dynamodb.DynamoDbAsyncClient; +import software.amazon.awssdk.services.dynamodb.model.ReturnConsumedCapacity; + +/** + * @author Alexander Patrikalakis + */ +public abstract class AbstractDynamoDBReadOperation extends AbstractDynamoDBDataPlaneOperation { + protected final boolean consistentRead; + + protected AbstractDynamoDBReadOperation(DataGenerator dataGenerator, DynamoDbAsyncClient dynamoDB, String tableName, + String partitionKeyName, boolean consistentRead, + ReturnConsumedCapacity returnConsumedCapacity, String operationName) { + super(dynamoDB, tableName, partitionKeyName, dataGenerator, returnConsumedCapacity, operationName); + this.consistentRead = consistentRead; + } +} diff --git a/ndbench-dynamodb-plugins/src/main/java/com/netflix/ndbench/plugin/dynamodb/operations/v2/dynamodb/dataplane/DynamoDBReadBulk.java b/ndbench-dynamodb-plugins/src/main/java/com/netflix/ndbench/plugin/dynamodb/operations/v2/dynamodb/dataplane/DynamoDBReadBulk.java new file mode 100644 index 00000000..51743bfc --- /dev/null +++ b/ndbench-dynamodb-plugins/src/main/java/com/netflix/ndbench/plugin/dynamodb/operations/v2/dynamodb/dataplane/DynamoDBReadBulk.java @@ -0,0 +1,89 @@ +/* + * Copyright 2018 Netflix, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ +package com.netflix.ndbench.plugin.dynamodb.operations.v2.dynamodb.dataplane; + +import com.google.common.base.Preconditions; +import com.google.common.collect.ImmutableMap; +import com.netflix.ndbench.api.plugin.DataGenerator; +import software.amazon.awssdk.services.dynamodb.DynamoDbAsyncClient; +import software.amazon.awssdk.services.dynamodb.model.AttributeValue; +import software.amazon.awssdk.services.dynamodb.model.BatchGetItemRequest; +import software.amazon.awssdk.services.dynamodb.model.BatchGetItemResponse; +import software.amazon.awssdk.services.dynamodb.model.KeysAndAttributes; +import software.amazon.awssdk.services.dynamodb.model.ReturnConsumedCapacity; + +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.stream.Collectors; + +/** + * @author Alexander Patrikalakis + * @author ipapapa + */ +public class DynamoDBReadBulk extends AbstractDynamoDBReadOperation, List> { + public DynamoDBReadBulk(DataGenerator dataGenerator, DynamoDbAsyncClient dynamoDB, String tableName, + String partitionKeyName, boolean consistentRead, ReturnConsumedCapacity returnConsumedCapacity) { + super(dataGenerator, dynamoDB, tableName, partitionKeyName, consistentRead, returnConsumedCapacity, + "BatchGetItem"); + } + + @Override + public List apply(List keys) { + Preconditions.checkArgument(new HashSet<>(keys).size() == keys.size()); + final KeysAndAttributes keysAndAttributes = generateReadRequests(keys); + readUntilDone(keysAndAttributes); + return keysAndAttributes.keys().stream() + .map(Map::toString) + .collect(Collectors.toList()); + } + + @Override + public BatchGetItemResponse measureConsumedCapacity(BatchGetItemResponse result) { + if (result.consumedCapacity() != null) { + consumed.addAndGet(getConsumedCapacityForTable(result.consumedCapacity())); + } + return result; + } + + private KeysAndAttributes generateReadRequests(List keys) { + return KeysAndAttributes.builder().keys(keys.stream() + .map(key -> ImmutableMap.of("id", AttributeValue.builder().s(key).build())) + .collect(Collectors.toList())) + .consistentRead(consistentRead).build(); + } + + private void readUntilDone(KeysAndAttributes keysAndAttributes) { + KeysAndAttributes remainingKeys = keysAndAttributes; + BatchGetItemResponse response; + do { + response = runBatchGetRequest(remainingKeys.toBuilder().consistentRead(consistentRead).build()); + remainingKeys = response.unprocessedKeys().get(tableName); + } while (remainingKeys != null && remainingKeys.keys() != null && !remainingKeys.keys().isEmpty()); + } + + private BatchGetItemResponse runBatchGetRequest(KeysAndAttributes keysAndAttributes) { + // TODO self throttle and estimate size of requests + return Optional.of(BatchGetItemRequest.builder() + .requestItems(ImmutableMap.of(tableName, keysAndAttributes)).build()) + .map(dynamoDB::batchGetItem) + .map(super::completeAndCatch) + .map(this::measureConsumedCapacity) + .orElse(null); + } +} diff --git a/ndbench-dynamodb-plugins/src/main/java/com/netflix/ndbench/plugin/dynamodb/operations/v2/dynamodb/dataplane/DynamoDBReadSingle.java b/ndbench-dynamodb-plugins/src/main/java/com/netflix/ndbench/plugin/dynamodb/operations/v2/dynamodb/dataplane/DynamoDBReadSingle.java new file mode 100644 index 00000000..72994668 --- /dev/null +++ b/ndbench-dynamodb-plugins/src/main/java/com/netflix/ndbench/plugin/dynamodb/operations/v2/dynamodb/dataplane/DynamoDBReadSingle.java @@ -0,0 +1,64 @@ +/* + * Copyright 2018 Netflix, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ +package com.netflix.ndbench.plugin.dynamodb.operations.v2.dynamodb.dataplane; + +import com.google.common.collect.ImmutableMap; +import com.netflix.ndbench.api.plugin.DataGenerator; +import software.amazon.awssdk.services.dynamodb.DynamoDbAsyncClient; +import software.amazon.awssdk.services.dynamodb.model.AttributeValue; +import software.amazon.awssdk.services.dynamodb.model.GetItemRequest; +import software.amazon.awssdk.services.dynamodb.model.GetItemResponse; +import software.amazon.awssdk.services.dynamodb.model.ReturnConsumedCapacity; + +import java.util.Map; +import java.util.Optional; + +/** + * @author Alexander Patrikalakis + * @author ipapapa + */ +public class DynamoDBReadSingle extends AbstractDynamoDBReadOperation { + public DynamoDBReadSingle(DataGenerator dataGenerator, DynamoDbAsyncClient dynamoDB, String tableName, + String partitionKeyName, boolean consistentRead, + ReturnConsumedCapacity returnConsumedCapacity) { + super(dataGenerator, dynamoDB, tableName, partitionKeyName, consistentRead, returnConsumedCapacity, "GetItem"); + } + + @Override + public String apply(String key) { + final GetItemRequest request = GetItemRequest.builder() + .key(ImmutableMap.of(partitionKeyName, AttributeValue.builder().s(key).build())) + .returnConsumedCapacity(returnConsumedCapacity) + .consistentRead(consistentRead) + .tableName(tableName) + .build(); + return Optional.ofNullable(dynamoDB.getItem(request)) + .map(super::completeAndCatch) + .map(this::measureConsumedCapacity) + .map(GetItemResponse::item) + .map(Map::toString) + .orElse(null); + } + + @Override + public GetItemResponse measureConsumedCapacity(GetItemResponse result) { + if (result.consumedCapacity() != null) { + consumed.addAndGet(result.consumedCapacity().capacityUnits()); + } + return result; + } +} diff --git a/ndbench-dynamodb-plugins/src/main/java/com/netflix/ndbench/plugin/dynamodb/operations/v2/dynamodb/dataplane/DynamoDBWriteBulk.java b/ndbench-dynamodb-plugins/src/main/java/com/netflix/ndbench/plugin/dynamodb/operations/v2/dynamodb/dataplane/DynamoDBWriteBulk.java new file mode 100644 index 00000000..0e3f3bee --- /dev/null +++ b/ndbench-dynamodb-plugins/src/main/java/com/netflix/ndbench/plugin/dynamodb/operations/v2/dynamodb/dataplane/DynamoDBWriteBulk.java @@ -0,0 +1,90 @@ +/* + * Copyright 2018 Netflix, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ +package com.netflix.ndbench.plugin.dynamodb.operations.v2.dynamodb.dataplane; + +import com.google.common.base.Preconditions; +import com.google.common.collect.ImmutableMap; +import com.netflix.ndbench.api.plugin.DataGenerator; +import software.amazon.awssdk.services.dynamodb.DynamoDbAsyncClient; +import software.amazon.awssdk.services.dynamodb.model.AttributeValue; +import software.amazon.awssdk.services.dynamodb.model.BatchWriteItemRequest; +import software.amazon.awssdk.services.dynamodb.model.BatchWriteItemResponse; +import software.amazon.awssdk.services.dynamodb.model.PutRequest; +import software.amazon.awssdk.services.dynamodb.model.ReturnConsumedCapacity; +import software.amazon.awssdk.services.dynamodb.model.WriteRequest; + +import java.util.HashSet; +import java.util.List; +import java.util.Optional; +import java.util.stream.Collectors; + +/** + * @author Alexander Patrikalakis + * @author ipapapa + */ +public class DynamoDBWriteBulk extends AbstractDynamoDBDataPlaneOperation, List> { + public DynamoDBWriteBulk(DataGenerator dataGenerator, DynamoDbAsyncClient dynamoDB, String tableName, + String partitionKeyName, ReturnConsumedCapacity returnConsumedCapacity) { + super(dynamoDB, tableName, partitionKeyName, dataGenerator, returnConsumedCapacity, "BatchWriteItem"); + } + + @Override + public List apply(List keys) { + Preconditions.checkArgument(new HashSet<>(keys).size() == keys.size()); + final List writeRequests = generateWriteRequests(keys); + writeUntilDone(writeRequests); + return writeRequests.stream() + .map(WriteRequest::putRequest) + .map(PutRequest::toString) + .collect(Collectors.toList()); + } + + @Override + public BatchWriteItemResponse measureConsumedCapacity(BatchWriteItemResponse result) { + if (result.consumedCapacity() != null) { + consumed.addAndGet(getConsumedCapacityForTable(result.consumedCapacity())); + } + return result; + } + + private List generateWriteRequests(List keys) { + return keys.stream() + .map(key -> ImmutableMap.of(partitionKeyName, AttributeValue.builder().s(key).build(), + ATTRIBUTE_NAME, AttributeValue.builder().s(this.dataGenerator.getRandomValue()).build())) + .map(item -> PutRequest.builder().item(item).build()) + .map(put -> WriteRequest.builder().putRequest(put).build()) + .collect(Collectors.toList()); + } + + private void writeUntilDone(List requests) { + List remainingRequests = requests; + BatchWriteItemResponse result; + do { + result = runBatchWriteRequest(remainingRequests); + remainingRequests = result.unprocessedItems().get(tableName); + } while (remainingRequests!= null && remainingRequests.isEmpty()); + } + + private BatchWriteItemResponse runBatchWriteRequest(List writeRequests) { + //todo self throttle + return Optional.of(BatchWriteItemRequest.builder().requestItems(ImmutableMap.of(tableName, writeRequests)).build()) + .map(dynamoDB::batchWriteItem) + .map(super::completeAndCatch) + .map(this::measureConsumedCapacity) + .orElse(null); + } +} diff --git a/ndbench-dynamodb-plugins/src/main/java/com/netflix/ndbench/plugin/dynamodb/operations/v2/dynamodb/dataplane/DynamoDBWriteSingle.java b/ndbench-dynamodb-plugins/src/main/java/com/netflix/ndbench/plugin/dynamodb/operations/v2/dynamodb/dataplane/DynamoDBWriteSingle.java new file mode 100644 index 00000000..56d457c4 --- /dev/null +++ b/ndbench-dynamodb-plugins/src/main/java/com/netflix/ndbench/plugin/dynamodb/operations/v2/dynamodb/dataplane/DynamoDBWriteSingle.java @@ -0,0 +1,60 @@ +/* + * Copyright 2018 Netflix, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ +package com.netflix.ndbench.plugin.dynamodb.operations.v2.dynamodb.dataplane; + + +import com.google.common.collect.ImmutableMap; +import com.netflix.ndbench.api.plugin.DataGenerator; +import software.amazon.awssdk.services.dynamodb.DynamoDbAsyncClient; +import software.amazon.awssdk.services.dynamodb.model.AttributeValue; +import software.amazon.awssdk.services.dynamodb.model.PutItemRequest; +import software.amazon.awssdk.services.dynamodb.model.PutItemResponse; +import software.amazon.awssdk.services.dynamodb.model.ReturnConsumedCapacity; + +import java.util.Optional; + +/** + * @author Alexander Patrikalakis + * @author ipapapa + */ +public class DynamoDBWriteSingle extends AbstractDynamoDBDataPlaneOperation { + public DynamoDBWriteSingle(DataGenerator dataGenerator, DynamoDbAsyncClient dynamoDB, String tableName, + String partitionKeyName, ReturnConsumedCapacity returnConsumedCapacity) { + super(dynamoDB, tableName, partitionKeyName, dataGenerator, returnConsumedCapacity, "PutItem"); + } + + @Override + public String apply(String key) { + // Write the item to the table + return Optional.ofNullable(dynamoDB.putItem(PutItemRequest.builder() + .tableName(tableName) + .item(ImmutableMap.of(partitionKeyName, AttributeValue.builder().s(key).build(), + ATTRIBUTE_NAME, AttributeValue.builder().s(dataGenerator.getRandomValue()).build())).build())) + .map(super::completeAndCatch) + .map(this::measureConsumedCapacity) + .map(PutItemResponse::toString) + .orElse(null); + } + + @Override + public PutItemResponse measureConsumedCapacity(PutItemResponse result) { + if (result.consumedCapacity() != null) { + consumed.addAndGet(result.consumedCapacity().capacityUnits()); + } + return result; + } +} diff --git a/ndbench-dynamodb-plugins/src/test/java/com/netflix/ndbench/plugin/dynamodb/DynamoDBProgrammaticKeyValueTest.java b/ndbench-dynamodb-plugins/src/test/java/com/netflix/ndbench/plugin/dynamodb/DynamoDBProgrammaticKeyValueTest.java index c5880862..76478f4c 100644 --- a/ndbench-dynamodb-plugins/src/test/java/com/netflix/ndbench/plugin/dynamodb/DynamoDBProgrammaticKeyValueTest.java +++ b/ndbench-dynamodb-plugins/src/test/java/com/netflix/ndbench/plugin/dynamodb/DynamoDBProgrammaticKeyValueTest.java @@ -9,6 +9,7 @@ import com.netflix.archaius.test.TestPropertyOverride; import com.netflix.governator.guice.test.ModulesForTesting; import com.netflix.governator.guice.test.junit4.GovernatorJunit4ClassRunner; +import com.netflix.ndbench.aws.defaultimpl.AwsDefaultsModule; import com.netflix.ndbench.core.defaultimpl.NdBenchGuiceModule; import com.netflix.ndbench.core.generators.DefaultDataGenerator; import com.netflix.ndbench.plugin.dynamodb.configs.DynamoDBModule; @@ -28,7 +29,7 @@ import static org.mockito.Mockito.verify; @RunWith(GovernatorJunit4ClassRunner.class) -@ModulesForTesting({DynamoDBModule.class, ArchaiusModule.class, NdBenchGuiceModule.class}) +@ModulesForTesting({DynamoDBModule.class, ArchaiusModule.class, NdBenchGuiceModule.class, AwsDefaultsModule.class}) public class DynamoDBProgrammaticKeyValueTest { @Inject ProgrammaticDynamoDBConfiguration configuration; diff --git a/ndbench-dynamodb-plugins/src/test/java/com/netflix/ndbench/plugin/dynamodb/configs/DynamoDBConfigurationTest.java b/ndbench-dynamodb-plugins/src/test/java/com/netflix/ndbench/plugin/dynamodb/configs/DynamoDBConfigurationTest.java index b150fa3d..bf575771 100644 --- a/ndbench-dynamodb-plugins/src/test/java/com/netflix/ndbench/plugin/dynamodb/configs/DynamoDBConfigurationTest.java +++ b/ndbench-dynamodb-plugins/src/test/java/com/netflix/ndbench/plugin/dynamodb/configs/DynamoDBConfigurationTest.java @@ -19,6 +19,7 @@ import com.netflix.archaius.guice.ArchaiusModule; import com.netflix.governator.guice.test.ModulesForTesting; import com.netflix.governator.guice.test.junit4.GovernatorJunit4ClassRunner; +import com.netflix.ndbench.aws.defaultimpl.AwsDefaultsModule; import org.junit.Test; import org.junit.runner.RunWith; @@ -27,7 +28,7 @@ import static org.junit.Assert.*; @RunWith(GovernatorJunit4ClassRunner.class) -@ModulesForTesting({DynamoDBModule.class, ArchaiusModule.class}) +@ModulesForTesting({DynamoDBModule.class, ArchaiusModule.class, AwsDefaultsModule.class}) public class DynamoDBConfigurationTest { @Inject DynamoDBConfiguration configuration; diff --git a/ndbench-dynamodb-plugins/src/test/java/com/netflix/ndbench/plugin/dynamodb/configs/ProgrammaticDynamoDBConfigurationTest.java b/ndbench-dynamodb-plugins/src/test/java/com/netflix/ndbench/plugin/dynamodb/configs/ProgrammaticDynamoDBConfigurationTest.java index d27ca0ea..4ed271fc 100644 --- a/ndbench-dynamodb-plugins/src/test/java/com/netflix/ndbench/plugin/dynamodb/configs/ProgrammaticDynamoDBConfigurationTest.java +++ b/ndbench-dynamodb-plugins/src/test/java/com/netflix/ndbench/plugin/dynamodb/configs/ProgrammaticDynamoDBConfigurationTest.java @@ -3,6 +3,7 @@ import com.netflix.archaius.guice.ArchaiusModule; import com.netflix.governator.guice.test.ModulesForTesting; import com.netflix.governator.guice.test.junit4.GovernatorJunit4ClassRunner; +import com.netflix.ndbench.aws.defaultimpl.AwsDefaultsModule; import org.junit.Test; import org.junit.runner.RunWith; @@ -11,7 +12,7 @@ import static org.junit.Assert.*; @RunWith(GovernatorJunit4ClassRunner.class) -@ModulesForTesting({DynamoDBModule.class, ArchaiusModule.class}) +@ModulesForTesting({DynamoDBModule.class, ArchaiusModule.class, AwsDefaultsModule.class}) public class ProgrammaticDynamoDBConfigurationTest { @Inject ProgrammaticDynamoDBConfiguration configuration; diff --git a/ndbench-dynamodb-plugins/src/test/java/com/netflix/ndbench/plugin/dynamodb/operations/controlplane/CreateTableTest.java b/ndbench-dynamodb-plugins/src/test/java/com/netflix/ndbench/plugin/dynamodb/operations/controlplane/CreateTableTest.java index 8c85962f..ba82f145 100644 --- a/ndbench-dynamodb-plugins/src/test/java/com/netflix/ndbench/plugin/dynamodb/operations/controlplane/CreateTableTest.java +++ b/ndbench-dynamodb-plugins/src/test/java/com/netflix/ndbench/plugin/dynamodb/operations/controlplane/CreateTableTest.java @@ -24,7 +24,7 @@ import com.amazonaws.services.dynamodbv2.model.ResourceNotFoundException; import com.amazonaws.services.dynamodbv2.model.TableDescription; import com.amazonaws.services.dynamodbv2.model.TableStatus; -import com.netflix.ndbench.plugin.dynamodb.operations.dynamodb.controlplane.CreateDynamoDBTable; +import com.netflix.ndbench.plugin.dynamodb.operations.v1.dynamodb.controlplane.CreateDynamoDBTable; import org.junit.Test; import org.mockito.Mockito; diff --git a/ndbench-dynamodb-plugins/src/test/java/com/netflix/ndbench/plugin/dynamodb/operations/controlplane/DeleteTableTest.java b/ndbench-dynamodb-plugins/src/test/java/com/netflix/ndbench/plugin/dynamodb/operations/controlplane/DeleteTableTest.java index 32d55f81..27ce26d8 100644 --- a/ndbench-dynamodb-plugins/src/test/java/com/netflix/ndbench/plugin/dynamodb/operations/controlplane/DeleteTableTest.java +++ b/ndbench-dynamodb-plugins/src/test/java/com/netflix/ndbench/plugin/dynamodb/operations/controlplane/DeleteTableTest.java @@ -22,7 +22,7 @@ import com.amazonaws.services.dynamodbv2.model.ResourceNotFoundException; import com.amazonaws.services.dynamodbv2.waiters.AmazonDynamoDBWaiters; import com.amazonaws.waiters.Waiter; -import com.netflix.ndbench.plugin.dynamodb.operations.dynamodb.controlplane.DeleteDynamoDBTable; +import com.netflix.ndbench.plugin.dynamodb.operations.v1.dynamodb.controlplane.DeleteDynamoDBTable; import org.junit.Test; import static org.junit.Assert.assertTrue;