Commit 2e28794d authored by njapke's avatar njapke Committed by pfandzelter
Browse files

Add local DynamoDB to nodeA in 3NodeTest

parent e9e2b6bb
Pipeline #128484 passed with stages
in 13 minutes and 55 seconds
......@@ -7,4 +7,4 @@
!tests/3NodeTest/pkg/
!tests/AlexandraTest/
!tests/consistency
!proto
\ No newline at end of file
!proto
......@@ -251,6 +251,10 @@ export AWS_PAGER=""
aws dynamodb delete-table --table-name fred
```
For debugging purposes, you can also set up a local instance of DynamoDB with the `amazon/dynamodb-local:1.16.0` Docker image.
To use that, use the `--dynamodb-endpoint` flag to point to your local endpoint.
You can also use the `--dynamodb-create-table` flag to have FReD create your DynamoDB table, yet that is not recommended, e.g., when multiple FReD machines share a table.
#### Remote
Instead of accessing the storage backends directly, all storage backends can also be accessed through gRPC, which makes it possible to run the storage backend service separately.
......@@ -553,7 +557,7 @@ All tests require Docker and Docker Compose to work.
The "3 node test" starts a FReD deployment of three FReD nodes and runs a client against the FReD cluster that validates different functionalities.
It uses Docker compose and can thus easily be started with `make 3n-all`.
The deployment comprises a single `etcd` Docker container as a NaSe, a simple trigger node, two FReD nodes that each comprise only a single machine (node _B_ and _C_) with a storage server, and a distributed FReD node _A_ that comprises three individual FReD machines behind a `fredproxy` sharing a single storage server.
The deployment comprises a single `etcd` Docker container as a NaSe, a simple trigger node, two FReD nodes that each comprise only a single machine (node _B_ and _C_) with a storage server, and a distributed FReD node _A_ that comprises three individual FReD machines behind a `fredproxy` sharing a single DynamoDB storage server.
All machines are connected over a Docker network.
The test client runs a number of operations against the FReD deployment and outputs a list of errors.
......
......@@ -67,10 +67,12 @@ type fredConfig struct {
CA string `env:"REMOTE_STORAGE_CA"`
}
DynamoDB struct {
Table string `env:"DYNAMODB_TABLE"`
Region string `env:"DYNAMODB_REGION"`
PublicKey string `env:"DYNAMODB_PUBLIC_KEY"`
PrivateKey string `env:"DYNAMODB_PRIVATE_KEY"`
Table string `env:"DYNAMODB_TABLE"`
Region string `env:"DYNAMODB_REGION"`
PublicKey string `env:"DYNAMODB_PUBLIC_KEY"`
PrivateKey string `env:"DYNAMODB_PRIVATE_KEY"`
Endpoint string `env:"DYNAMODB_ENDPOINT"`
CreateTable bool `env:"DYNAMODB_CREATETABLE"`
}
Bdb struct {
Path string `env:"BADGERDB_PATH"`
......@@ -123,6 +125,8 @@ func parseArgs() (fc fredConfig) {
flag.StringVar(&(fc.DynamoDB.Region), "dynamo-region", "", "AWS region for DynamoDB storage backend. (Env: DYNAMODB_REGION)")
flag.StringVar(&(fc.DynamoDB.PublicKey), "dynamo-public-key", "", "AWS public key for DynamoDB storage backend. (Env: DYNAMODB_PUBLIC_KEY)")
flag.StringVar(&(fc.DynamoDB.PrivateKey), "dynamo-private-key", "", "AWS private key for DynamoDB storage backend. (Env: DYNAMODB_PRIVATE_KEY)")
flag.StringVar(&(fc.DynamoDB.Endpoint), "dynamo-endpoint", "", "Endpoint of local DynamoDB instance. Leave empty to use DynamoDB on AWS. (Env: DYNAMODB_ENDPOINT)")
flag.BoolVar(&(fc.DynamoDB.CreateTable), "dynamo-create-table", false, "Create table in DynamoDB on first run rather than using existing table. (Env: DYNAMODB_CREATETABLE)")
flag.StringVar(&(fc.Bdb.Path), "badgerdb-path", "", "Path to the BadgerDB database. (Env: BADGERDB_PATH)")
......@@ -273,7 +277,7 @@ func main() {
case "remote":
store = storageclient.NewClient(fc.RemoteStore.Host, fc.RemoteStore.Cert, fc.RemoteStore.Key, strings.Split(fc.RemoteStore.CA, ","))
case "dynamo":
store, err = dynamo.New(fc.DynamoDB.Table, fc.DynamoDB.Region)
store, err = dynamo.New(fc.DynamoDB.Table, fc.DynamoDB.Region, fc.DynamoDB.Endpoint, fc.DynamoDB.CreateTable)
if err != nil {
log.Fatal().Msgf("could not open a dynamo connection: %s", err.(*errors.Error).ErrorStack())
}
......
......@@ -3,6 +3,7 @@ package dynamo
import (
"context"
"encoding/json"
"fmt"
"net/http"
"net/url"
"sort"
......@@ -13,6 +14,7 @@ import (
"github.com/DistributedClocks/GoVector/govec/vclock"
"github.com/aws/aws-sdk-go-v2/aws"
shttp "github.com/aws/aws-sdk-go-v2/aws/transport/http"
"github.com/aws/aws-sdk-go-v2/credentials"
"github.com/aws/aws-sdk-go-v2/feature/dynamodb/expression"
"github.com/aws/aws-sdk-go-v2/service/dynamodb"
dynamoDBTypes "github.com/aws/aws-sdk-go-v2/service/dynamodb/types"
......@@ -80,7 +82,91 @@ func vectorToString(v vclock.VClock) string {
return url.QueryEscape(vector.SortedVCString(v))
}
func NewFromExisting(table string, svc *dynamodb.Client) (s *Storage, err error) {
func New(table string, region string, endpoint string, create bool) (*Storage, error) {
log.Debug().Msgf("creating a new dynamodb connection to table %s in region %s", table, region)
log.Debug().Msg("Checked creds - OK!")
opts := dynamodb.Options{
Region: region,
}
if endpoint != "" {
log.Debug().Msgf("creating a new local dynamodb connection (endpoint: http://%s) to table %s in region %s", endpoint, table, region)
creds := credentials.NewStaticCredentialsProvider("TEST_KEY", "TEST_SECRET", "")
opts = dynamodb.Options{
Region: region,
Credentials: creds,
EndpointResolver: dynamodb.EndpointResolverFromURL(fmt.Sprintf("http://%s", endpoint)),
}
}
log.Debug().Msg("Created session - OK!")
svc := dynamodb.New(opts)
log.Debug().Msg("Created service - OK!")
if create {
// aws dynamodb create-table --table-name fred --attribute-definitions "AttributeName=Keygroup,AttributeType=S AttributeName=Key,AttributeType=S" --key-schema "AttributeName=Keygroup,KeyType=HASH AttributeName=Key,KeyType=RANGE" --provisioned-throughput "ReadCapacityUnits=1,WriteCapacityUnits=1"
_, err := svc.CreateTable(context.TODO(), &dynamodb.CreateTableInput{
AttributeDefinitions: []dynamoDBTypes.AttributeDefinition{
{
AttributeName: aws.String(keygroupName),
AttributeType: dynamoDBTypes.ScalarAttributeTypeS,
},
{
AttributeName: aws.String(keyName),
AttributeType: dynamoDBTypes.ScalarAttributeTypeS,
},
},
GlobalSecondaryIndexes: nil,
KeySchema: []dynamoDBTypes.KeySchemaElement{
{
AttributeName: aws.String(keygroupName),
KeyType: dynamoDBTypes.KeyTypeHash,
},
{
AttributeName: aws.String(keyName),
KeyType: dynamoDBTypes.KeyTypeRange,
},
},
ProvisionedThroughput: &dynamoDBTypes.ProvisionedThroughput{
ReadCapacityUnits: aws.Int64(1),
WriteCapacityUnits: aws.Int64(1),
},
TableName: aws.String(table),
})
if err != nil {
return nil, err
}
log.Debug().Msg("DynamoDB: Created table - OK!")
// aws dynamodb update-time-to-live --table-name fred --time-to-live-specification "Enabled=true, AttributeName=Expiry"
_, err = svc.UpdateTimeToLive(context.TODO(), &dynamodb.UpdateTimeToLiveInput{
TableName: aws.String(table),
TimeToLiveSpecification: &dynamoDBTypes.TimeToLiveSpecification{
AttributeName: aws.String(expiryKey),
Enabled: aws.Bool(true),
},
})
if err != nil {
return nil, err
}
log.Debug().Msg("DynamoDB: Configured TTL - OK!")
return &Storage{
dynamotable: table,
svc: svc,
}, nil
}
log.Debug().Msgf("Loading table %s...", table)
// check if the table with that name even exists
// if not: error out
......@@ -119,25 +205,6 @@ func NewFromExisting(table string, svc *dynamodb.Client) (s *Storage, err error)
}, nil
}
// New creates a new Session for DynamoDB.
func New(table, region string) (s *Storage, err error) {
log.Debug().Msgf("creating a new dynamodb connection to table %s in region %s", table, region)
log.Debug().Msg("Checked creds - OK!")
opts := dynamodb.Options{
Region: region,
}
log.Debug().Msg("Created session - OK!")
svc := dynamodb.New(opts)
log.Debug().Msg("Created service - OK!")
return NewFromExisting(table, svc)
}
// Close closes the underlying DynamoDB connection (no cleanup needed at the moment).
func (s *Storage) Close() error {
return nil
......
......@@ -9,11 +9,6 @@ import (
"time"
"github.com/DistributedClocks/GoVector/govec/vclock"
"github.com/aws/aws-sdk-go-v2/aws"
"github.com/aws/aws-sdk-go-v2/config"
"github.com/aws/aws-sdk-go-v2/credentials"
"github.com/aws/aws-sdk-go-v2/service/dynamodb"
dynamoDBTypes "github.com/aws/aws-sdk-go-v2/service/dynamodb/types"
"github.com/rs/zerolog"
"github.com/rs/zerolog/log"
"github.com/stretchr/testify/assert"
......@@ -73,91 +68,7 @@ func TestMain(m *testing.M) {
log.Fatal().Msg(err.Error())
}
cfg, err := config.LoadDefaultConfig(context.TODO(),
config.WithRegion("eu-central-1"),
)
if err != nil {
log.Fatal().Msg(err.Error())
return
}
cfg.Credentials = credentials.NewStaticCredentialsProvider("TEST_KEY", "TEST_SECRET", "")
svc := dynamodb.New(dynamodb.Options{
Region: cfg.Region,
HTTPClient: cfg.HTTPClient,
Credentials: cfg.Credentials,
APIOptions: cfg.APIOptions,
Logger: cfg.Logger,
ClientLogMode: cfg.ClientLogMode,
EndpointResolver: dynamodb.EndpointResolverFromURL(fmt.Sprintf("http://%s:%s", ip, port)),
})
log.Debug().Msg("Created session - OK!")
// aws dynamodb create-table --table-name fred --attribute-definitions "AttributeName=Keygroup,AttributeType=S AttributeName=Key,AttributeType=S" --key-schema "AttributeName=Keygroup,KeyType=HASH AttributeName=Key,KeyType=RANGE" --provisioned-throughput "ReadCapacityUnits=1,WriteCapacityUnits=1"
out1, err := svc.CreateTable(context.TODO(), &dynamodb.CreateTableInput{
AttributeDefinitions: []dynamoDBTypes.AttributeDefinition{
{
AttributeName: aws.String(keygroupName),
AttributeType: dynamoDBTypes.ScalarAttributeTypeS,
},
{
AttributeName: aws.String(keyName),
AttributeType: dynamoDBTypes.ScalarAttributeTypeS,
},
},
GlobalSecondaryIndexes: nil,
KeySchema: []dynamoDBTypes.KeySchemaElement{
{
AttributeName: aws.String(keygroupName),
KeyType: dynamoDBTypes.KeyTypeHash,
},
{
AttributeName: aws.String(keyName),
KeyType: dynamoDBTypes.KeyTypeRange,
},
},
ProvisionedThroughput: &dynamoDBTypes.ProvisionedThroughput{
ReadCapacityUnits: aws.Int64(1),
WriteCapacityUnits: aws.Int64(1),
},
TableName: aws.String(table),
})
if err != nil {
log.Fatal().Msg(err.Error())
return
}
if out1 != nil {
log.Debug().Msgf("Creating table, output: %+v", out1)
}
log.Debug().Msg("Created table - OK!")
// aws dynamodb update-time-to-live --table-name fred --time-to-live-specification "Enabled=true, AttributeName=Expiry"
out2, err := svc.UpdateTimeToLive(context.TODO(), &dynamodb.UpdateTimeToLiveInput{
TableName: aws.String(table),
TimeToLiveSpecification: &dynamoDBTypes.TimeToLiveSpecification{
AttributeName: aws.String(expiryKey),
Enabled: aws.Bool(true),
},
})
if err != nil {
log.Fatal().Msg(err.Error())
return
}
if out2 != nil {
log.Debug().Msgf("Updating TTL on table, output: %+v", out2)
}
log.Debug().Msg("Configured TTL - OK!")
db, err = NewFromExisting(table, svc)
db, err = New(table, "eu-central-1", fmt.Sprintf("%s:%s", ip, port), true)
if err != nil {
log.Fatal().Msg(err.Error())
......
**/*
!local-fred-dynamo.db
!/pkg/*
!/cmd/*
!nodeA-haproxy.cfg
FROM amazon/dynamodb-local:1.16.0
WORKDIR /home/dynamodblocal
# add prepared database file to local dynamodb image
RUN mkdir data
COPY --chown=dynamodblocal:dynamodblocal local-fred-dynamo.db data/shared-local-instance.db
RUN chmod -R 777 data
EXPOSE 8000
ENTRYPOINT [ "java", "-jar", "DynamoDBLocal.jar", "-sharedDb", "-dbPath", "./data" ]
\ No newline at end of file
......@@ -5,7 +5,7 @@ export TEST_RANGE := -
all: clean 3n-all clean
clean: ## clean up all resources
@docker-compose -f etcd.yml -f nodeA.yml -f nodeB.yml -f nodeC.yml -f 3NodeTester.yml -f trigger.yml down --remove-orphan
@docker-compose -f etcd.yml -f nodeA.yml -f nodeB.yml -f nodeC.yml -f 3NodeTester.yml -f trigger.yml down --remove-orphans
@docker network remove fredwork || true
3n-standard: export TEST_RANGE = 1-1
......
......@@ -36,7 +36,7 @@ services:
build: ..
image: fred/fred:local
container_name: nodeA-1
command: "--remote-storage-host 172.26.1.104:1337 \
command: "--dynamo-endpoint 172.26.1.104:8000 \
--peer-host 172.26.1.101:5555 \
--peer-host-proxy 172.26.1.1:5555 \
--nodeID nodeA \
......@@ -48,7 +48,9 @@ services:
--peer-cert /cert/nodeA.crt \
--peer-key /cert/nodeA.key \
--peer-ca /cert/ca.crt \
--adaptor remote \
--adaptor dynamo \
--dynamo-table fred \
--dynamo-region eu-central-1 \
--cpuprofile /profiles/cpuprofile.pprof \
--memprofile /profiles/memprofile.pprof \
--nase-host https://172.26.6.1:2379 \
......@@ -57,10 +59,6 @@ services:
--nase-ca /cert/ca.crt \
--nase-cached \
--handler dev \
--badgerdb-path ./db \
--remote-storage-cert /cert/nodeA.crt \
--remote-storage-key /cert/nodeA.key \
--remote-storage-ca /cert/ca.crt \
--trigger-cert /cert/nodeA.crt \
--trigger-key /cert/nodeA.key \
--trigger-ca /cert/ca.crt"
......@@ -83,7 +81,7 @@ services:
build: ..
image: fred/fred:local
container_name: nodeA-2
command: "--remote-storage-host 172.26.1.104:1337 \
command: "--dynamo-endpoint 172.26.1.104:8000 \
--peer-host 172.26.1.102:5555 \
--peer-host-proxy 172.26.1.1:5555 \
--nodeID nodeA \
......@@ -95,17 +93,15 @@ services:
--peer-cert /cert/nodeA.crt \
--peer-key /cert/nodeA.key \
--peer-ca /cert/ca.crt \
--adaptor remote \
--adaptor dynamo \
--dynamo-table fred \
--dynamo-region eu-central-1 \
--nase-host https://172.26.6.1:2379 \
--nase-cert /cert/nodeA.crt \
--nase-key /cert/nodeA.key \
--nase-ca /cert/ca.crt \
--nase-cached \
--handler dev \
--badgerdb-path ./db \
--remote-storage-cert /cert/nodeA.crt \
--remote-storage-key /cert/nodeA.key \
--remote-storage-ca /cert/ca.crt \
--trigger-cert /cert/nodeA.crt \
--trigger-key /cert/nodeA.key \
--trigger-ca /cert/ca.crt"
......@@ -126,7 +122,7 @@ services:
build: ..
image: fred/fred:local
container_name: nodeA-3
command: "--remote-storage-host 172.26.1.104:1337 \
command: "--dynamo-endpoint 172.26.1.104:8000 \
--peer-host 172.26.1.103:5555 \
--peer-host-proxy 172.26.1.1:5555 \
--nodeID nodeA \
......@@ -138,17 +134,15 @@ services:
--peer-cert /cert/nodeA.crt \
--peer-key /cert/nodeA.key \
--peer-ca /cert/ca.crt \
--adaptor remote \
--adaptor dynamo \
--dynamo-table fred \
--dynamo-region eu-central-1 \
--nase-host https://172.26.6.1:2379 \
--nase-cert /cert/nodeA.crt \
--nase-key /cert/nodeA.key \
--nase-ca /cert/ca.crt \
--nase-cached \
--handler dev \
--badgerdb-path ./db \
--remote-storage-cert /cert/nodeA.crt \
--remote-storage-key /cert/nodeA.key \
--remote-storage-ca /cert/ca.crt \
--trigger-cert /cert/nodeA.crt \
--trigger-key /cert/nodeA.key \
--trigger-ca /cert/ca.crt"
......@@ -164,18 +158,12 @@ services:
storeA:
build:
context: ..
dockerfile: storage.Dockerfile
image: fred/store:local
context: 3NodeTest
dockerfile: dynamodb.Dockerfile
image: "fred/dynamodb:local"
container_name: storeA
command: "--log-level '${LOG_LEVEL_STORE}' \
--cert /cert/cert.crt \
--key /cert/key.key \
--ca-file /cert/ca.crt"
volumes:
- ./certificates/storeA.crt:/cert/cert.crt
- ./certificates/storeA.key:/cert/key.key
- ./certificates/ca.crt:/cert/ca.crt
ports:
- "8000:8000"
networks:
fredwork:
ipv4_address: 172.26.1.104
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment