Commit 9a7e8a0a authored by pfandzelter's avatar pfandzelter
Browse files

cleanup

parent c7a3ccff
Pipeline #128443 passed with stages
in 18 minutes and 53 seconds
**/*
!/pkg/*
!/cmd/*
!/dynamodb-data/*
!go.mod
!go.sum
!tests/3NodeTest/cmd
......
......@@ -251,6 +251,10 @@ export AWS_PAGER=""
aws dynamodb delete-table --table-name fred
```
For debugging purposes, you can also set up a local instance of DynamoDB with the `amazon/dynamodb-local:1.16.0` Docker image.
To use that, use the `--dynamodb-endpoint` flag to point to your local endpoint.
You can also use the `--dynamodb-create-table` flag to have FReD create your DynamoDB table, yet that is not recommended, e.g., when multiple FReD machines share a table.
#### Remote
Instead of accessing the storage backends directly, all storage backends can also be accessed through gRPC, which makes it possible to run the storage backend service separately.
......@@ -553,7 +557,7 @@ All tests require Docker and Docker Compose to work.
The "3 node test" starts a FReD deployment of three FReD nodes and runs a client against the FReD cluster that validates different functionalities.
It uses Docker compose and can thus easily be started with `make 3n-all`.
The deployment comprises a single `etcd` Docker container as a NaSe, a simple trigger node, two FReD nodes that each comprise only a single machine (node _B_ and _C_) with a storage server, and a distributed FReD node _A_ that comprises three individual FReD machines behind a `fredproxy` sharing a single storage server.
The deployment comprises a single `etcd` Docker container as a NaSe, a simple trigger node, two FReD nodes that each comprise only a single machine (node _B_ and _C_) with a storage server, and a distributed FReD node _A_ that comprises three individual FReD machines behind a `fredproxy` sharing a single DynamoDB storage server.
All machines are connected over a Docker network.
The test client runs a number of operations against the FReD deployment and outputs a list of errors.
......
......@@ -67,11 +67,12 @@ type fredConfig struct {
CA string `env:"REMOTE_STORAGE_CA"`
}
DynamoDB struct {
Table string `env:"DYNAMODB_TABLE"`
Region string `env:"DYNAMODB_REGION"`
PublicKey string `env:"DYNAMODB_PUBLIC_KEY"`
PrivateKey string `env:"DYNAMODB_PRIVATE_KEY"`
Endpoint string `env:"DYNAMODB_ENDPOINT"`
Table string `env:"DYNAMODB_TABLE"`
Region string `env:"DYNAMODB_REGION"`
PublicKey string `env:"DYNAMODB_PUBLIC_KEY"`
PrivateKey string `env:"DYNAMODB_PRIVATE_KEY"`
Endpoint string `env:"DYNAMODB_ENDPOINT"`
CreateTable bool `env:"DYNAMODB_CREATETABLE"`
}
Bdb struct {
Path string `env:"BADGERDB_PATH"`
......@@ -125,6 +126,7 @@ func parseArgs() (fc fredConfig) {
flag.StringVar(&(fc.DynamoDB.PublicKey), "dynamo-public-key", "", "AWS public key for DynamoDB storage backend. (Env: DYNAMODB_PUBLIC_KEY)")
flag.StringVar(&(fc.DynamoDB.PrivateKey), "dynamo-private-key", "", "AWS private key for DynamoDB storage backend. (Env: DYNAMODB_PRIVATE_KEY)")
flag.StringVar(&(fc.DynamoDB.Endpoint), "dynamo-endpoint", "", "Endpoint of local DynamoDB instance. Leave empty to use DynamoDB on AWS. (Env: DYNAMODB_ENDPOINT)")
flag.BoolVar(&(fc.DynamoDB.CreateTable), "dynamo-create-table", false, "Create table in DynamoDB on first run rather than using existing table. (Env: DYNAMODB_CREATETABLE)")
flag.StringVar(&(fc.Bdb.Path), "badgerdb-path", "", "Path to the BadgerDB database. (Env: BADGERDB_PATH)")
......@@ -275,16 +277,9 @@ func main() {
case "remote":
store = storageclient.NewClient(fc.RemoteStore.Host, fc.RemoteStore.Cert, fc.RemoteStore.Key, strings.Split(fc.RemoteStore.CA, ","))
case "dynamo":
if fc.DynamoDB.Endpoint != "" {
store, err = dynamo.NewLocal(fc.DynamoDB.Table, fc.DynamoDB.Region, fc.DynamoDB.Endpoint)
if err != nil {
log.Fatal().Msgf("could not open a dynamo connection: %s", err.(*errors.Error).ErrorStack())
}
} else {
store, err = dynamo.New(fc.DynamoDB.Table, fc.DynamoDB.Region)
if err != nil {
log.Fatal().Msgf("could not open a dynamo connection: %s", err.(*errors.Error).ErrorStack())
}
store, err = dynamo.New(fc.DynamoDB.Table, fc.DynamoDB.Region, fc.DynamoDB.Endpoint, fc.DynamoDB.CreateTable)
if err != nil {
log.Fatal().Msgf("could not open a dynamo connection: %s", err.(*errors.Error).ErrorStack())
}
default:
log.Fatal().Msg("unknown storage backend")
......
......@@ -82,7 +82,91 @@ func vectorToString(v vclock.VClock) string {
return url.QueryEscape(vector.SortedVCString(v))
}
func NewFromExisting(table string, svc *dynamodb.Client) (s *Storage, err error) {
func New(table string, region string, endpoint string, create bool) (*Storage, error) {
log.Debug().Msgf("creating a new dynamodb connection to table %s in region %s", table, region)
log.Debug().Msg("Checked creds - OK!")
opts := dynamodb.Options{
Region: region,
}
if endpoint != "" {
log.Debug().Msgf("creating a new local dynamodb connection (endpoint: http://%s) to table %s in region %s", endpoint, table, region)
creds := credentials.NewStaticCredentialsProvider("TEST_KEY", "TEST_SECRET", "")
opts = dynamodb.Options{
Region: region,
Credentials: creds,
EndpointResolver: dynamodb.EndpointResolverFromURL(fmt.Sprintf("http://%s", endpoint)),
}
}
log.Debug().Msg("Created session - OK!")
svc := dynamodb.New(opts)
log.Debug().Msg("Created service - OK!")
if create {
// aws dynamodb create-table --table-name fred --attribute-definitions "AttributeName=Keygroup,AttributeType=S AttributeName=Key,AttributeType=S" --key-schema "AttributeName=Keygroup,KeyType=HASH AttributeName=Key,KeyType=RANGE" --provisioned-throughput "ReadCapacityUnits=1,WriteCapacityUnits=1"
_, err := svc.CreateTable(context.TODO(), &dynamodb.CreateTableInput{
AttributeDefinitions: []dynamoDBTypes.AttributeDefinition{
{
AttributeName: aws.String(keygroupName),
AttributeType: dynamoDBTypes.ScalarAttributeTypeS,
},
{
AttributeName: aws.String(keyName),
AttributeType: dynamoDBTypes.ScalarAttributeTypeS,
},
},
GlobalSecondaryIndexes: nil,
KeySchema: []dynamoDBTypes.KeySchemaElement{
{
AttributeName: aws.String(keygroupName),
KeyType: dynamoDBTypes.KeyTypeHash,
},
{
AttributeName: aws.String(keyName),
KeyType: dynamoDBTypes.KeyTypeRange,
},
},
ProvisionedThroughput: &dynamoDBTypes.ProvisionedThroughput{
ReadCapacityUnits: aws.Int64(1),
WriteCapacityUnits: aws.Int64(1),
},
TableName: aws.String(table),
})
if err != nil {
return nil, err
}
log.Debug().Msg("DynamoDB: Created table - OK!")
// aws dynamodb update-time-to-live --table-name fred --time-to-live-specification "Enabled=true, AttributeName=Expiry"
_, err = svc.UpdateTimeToLive(context.TODO(), &dynamodb.UpdateTimeToLiveInput{
TableName: aws.String(table),
TimeToLiveSpecification: &dynamoDBTypes.TimeToLiveSpecification{
AttributeName: aws.String(expiryKey),
Enabled: aws.Bool(true),
},
})
if err != nil {
return nil, err
}
log.Debug().Msg("DynamoDB: Configured TTL - OK!")
return &Storage{
dynamotable: table,
svc: svc,
}, nil
}
log.Debug().Msgf("Loading table %s...", table)
// check if the table with that name even exists
// if not: error out
......@@ -121,48 +205,6 @@ func NewFromExisting(table string, svc *dynamodb.Client) (s *Storage, err error)
}, nil
}
// New creates a new Session for DynamoDB.
func New(table, region string) (s *Storage, err error) {
log.Debug().Msgf("creating a new dynamodb connection to table %s in region %s", table, region)
log.Debug().Msg("Checked creds - OK!")
opts := dynamodb.Options{
Region: region,
}
log.Debug().Msg("Created session - OK!")
svc := dynamodb.New(opts)
log.Debug().Msg("Created service - OK!")
return NewFromExisting(table, svc)
}
// NewLocal creates a new Session for a local DynamoDB instance.
func NewLocal(table, region, endpoint string) (s *Storage, err error) {
log.Debug().Msgf("creating a new local dynamodb connection (endpoint: %s) to table %s in region %s", endpoint, table, region)
// log.Debug().Msg("Checked creds - OK!")
creds := credentials.NewStaticCredentialsProvider("TEST_KEY", "TEST_SECRET", "")
opts := dynamodb.Options{
Region: region,
Credentials: creds,
EndpointResolver: dynamodb.EndpointResolverFromURL(fmt.Sprintf("http://%s", endpoint)),
}
log.Debug().Msg("Created session - OK!")
svc := dynamodb.New(opts)
log.Debug().Msg("Created service - OK!")
return NewFromExisting(table, svc)
}
// Close closes the underlying DynamoDB connection (no cleanup needed at the moment).
func (s *Storage) Close() error {
return nil
......
......@@ -9,11 +9,6 @@ import (
"time"
"github.com/DistributedClocks/GoVector/govec/vclock"
"github.com/aws/aws-sdk-go-v2/aws"
"github.com/aws/aws-sdk-go-v2/config"
"github.com/aws/aws-sdk-go-v2/credentials"
"github.com/aws/aws-sdk-go-v2/service/dynamodb"
dynamoDBTypes "github.com/aws/aws-sdk-go-v2/service/dynamodb/types"
"github.com/rs/zerolog"
"github.com/rs/zerolog/log"
"github.com/stretchr/testify/assert"
......@@ -73,91 +68,7 @@ func TestMain(m *testing.M) {
log.Fatal().Msg(err.Error())
}
cfg, err := config.LoadDefaultConfig(context.TODO(),
config.WithRegion("eu-central-1"),
)
if err != nil {
log.Fatal().Msg(err.Error())
return
}
cfg.Credentials = credentials.NewStaticCredentialsProvider("TEST_KEY", "TEST_SECRET", "")
svc := dynamodb.New(dynamodb.Options{
Region: cfg.Region,
HTTPClient: cfg.HTTPClient,
Credentials: cfg.Credentials,
APIOptions: cfg.APIOptions,
Logger: cfg.Logger,
ClientLogMode: cfg.ClientLogMode,
EndpointResolver: dynamodb.EndpointResolverFromURL(fmt.Sprintf("http://%s:%s", ip, port)),
})
log.Debug().Msg("Created session - OK!")
// aws dynamodb create-table --table-name fred --attribute-definitions "AttributeName=Keygroup,AttributeType=S AttributeName=Key,AttributeType=S" --key-schema "AttributeName=Keygroup,KeyType=HASH AttributeName=Key,KeyType=RANGE" --provisioned-throughput "ReadCapacityUnits=1,WriteCapacityUnits=1"
out1, err := svc.CreateTable(context.TODO(), &dynamodb.CreateTableInput{
AttributeDefinitions: []dynamoDBTypes.AttributeDefinition{
{
AttributeName: aws.String(keygroupName),
AttributeType: dynamoDBTypes.ScalarAttributeTypeS,
},
{
AttributeName: aws.String(keyName),
AttributeType: dynamoDBTypes.ScalarAttributeTypeS,
},
},
GlobalSecondaryIndexes: nil,
KeySchema: []dynamoDBTypes.KeySchemaElement{
{
AttributeName: aws.String(keygroupName),
KeyType: dynamoDBTypes.KeyTypeHash,
},
{
AttributeName: aws.String(keyName),
KeyType: dynamoDBTypes.KeyTypeRange,
},
},
ProvisionedThroughput: &dynamoDBTypes.ProvisionedThroughput{
ReadCapacityUnits: aws.Int64(1),
WriteCapacityUnits: aws.Int64(1),
},
TableName: aws.String(table),
})
if err != nil {
log.Fatal().Msg(err.Error())
return
}
if out1 != nil {
log.Debug().Msgf("Creating table, output: %+v", out1)
}
log.Debug().Msg("Created table - OK!")
// aws dynamodb update-time-to-live --table-name fred --time-to-live-specification "Enabled=true, AttributeName=Expiry"
out2, err := svc.UpdateTimeToLive(context.TODO(), &dynamodb.UpdateTimeToLiveInput{
TableName: aws.String(table),
TimeToLiveSpecification: &dynamoDBTypes.TimeToLiveSpecification{
AttributeName: aws.String(expiryKey),
Enabled: aws.Bool(true),
},
})
if err != nil {
log.Fatal().Msg(err.Error())
return
}
if out2 != nil {
log.Debug().Msgf("Updating TTL on table, output: %+v", out2)
}
log.Debug().Msg("Configured TTL - OK!")
db, err = NewFromExisting(table, svc)
db, err = New(table, "eu-central-1", fmt.Sprintf("%s:%s", ip, port), true)
if err != nil {
log.Fatal().Msg(err.Error())
......
**/*
!local-fred-dynamo.db
!/pkg/*
!/cmd/*
!nodeA-haproxy.cfg
......@@ -3,9 +3,10 @@ FROM amazon/dynamodb-local:1.16.0
WORKDIR /home/dynamodblocal
# add prepared database file to local dynamodb image
COPY --chown=dynamodblocal:dynamodblocal dynamodb-data data
RUN mkdir data
COPY --chown=dynamodblocal:dynamodblocal local-fred-dynamo.db data/shared-local-instance.db
RUN chmod -R 777 data
EXPOSE 8000
ENTRYPOINT [ "java", "-jar", "DynamoDBLocal.jar", "-sharedDb", "-dbPath", "./data" ]
ENTRYPOINT [ "java", "-jar", "DynamoDBLocal.jar", "-sharedDb", "-dbPath", "./data" ]
\ No newline at end of file
......@@ -5,7 +5,7 @@ export TEST_RANGE := -
all: clean 3n-all clean
clean: ## clean up all resources
@docker-compose -f etcd.yml -f nodeA.yml -f nodeB.yml -f nodeC.yml -f 3NodeTester.yml -f trigger.yml down --remove-orphan
@docker-compose -f etcd.yml -f nodeA.yml -f nodeB.yml -f nodeC.yml -f 3NodeTester.yml -f trigger.yml down --remove-orphans
@docker network remove fredwork || true
3n-standard: export TEST_RANGE = 1-1
......
......@@ -158,7 +158,7 @@ services:
storeA:
build:
context: ..
context: 3NodeTest
dockerfile: dynamodb.Dockerfile
image: "fred/dynamodb:local"
container_name: storeA
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment