Commit 0dbc1f48 authored by pfandzelter's avatar pfandzelter
Browse files

improve 3nodetest for users

parent befb35df
Pipeline #15396 passed with stages
in 17 minutes and 30 seconds
......@@ -543,7 +543,7 @@ This is part of a TDD approach where tests can be defined first and the software
The "3 node test" starts a FReD deployment of three FReD nodes and runs a client against the FReD cluster that validates different functionalities.
It can be found in `./tests/3NodeTest`.
It uses Docker compose and can thus easily be started with `make 3nodetest`.
It uses Docker compose and can thus easily be started with `make 3n-all`.
The deployment comprises a single `etcd` Docker container as a NaSe, a simple trigger node, two FReD nodes that each comprise only a single machine (node _B_ and _C_) with a storage server, and a distributed FReD node _A_ that comprises three individual FReD machines behind a `haproxy` sharing a single storage server.
All machines are connected over a Docker network.
......
......@@ -16,7 +16,7 @@ import (
var (
// Wait for the user to press enter to continue
waitUser bool
waitUser *bool
reader = bufio.NewReader(os.Stdin)
)
......@@ -31,73 +31,73 @@ func main() {
)
// Parse Flags
waitUser = *flag.Bool("wait-user", false, "wait for user input after each test")
waitUser = flag.Bool("wait-user", false, "wait for user input after each test")
nodeAhost := *flag.String("nodeAhost", "172.26.1.1", "host of nodeA (e.g. localhost)") // Docker: localhost
nodeAhttpPort := *flag.String("nodeAhttp", "9001", "port of nodeA (e.g. 9001)") // Docker: 9002
nodeApeeringID := *flag.String("nodeAzmqID", "nodeA", "ZMQ Id of nodeA")
nodeAhost := flag.String("nodeAhost", "", "host of nodeA (e.g. localhost)") // Docker: localhost
nodeAhttpPort := flag.String("nodeAhttp", "", "port of nodeA (e.g. 9001)") // Docker: 9002
nodeApeeringID := flag.String("nodeApeeringID", "", "Peering Id of nodeA")
nodeBhost := *flag.String("nodeBhost", "172.26.2.1", "host of nodeB (e.g. localhost)")
nodeBhttpPort := *flag.String("nodeBhttp", "9001", "port of nodeB (e.g. 9001)")
nodeBpeeringID := *flag.String("nodeBzmqID", "nodeB", "ZMQ Id of nodeB")
nodeBhost := flag.String("nodeBhost", "", "host of nodeB (e.g. localhost)")
nodeBhttpPort := flag.String("nodeBhttp", "", "port of nodeB (e.g. 9001)")
nodeBpeeringID := flag.String("nodeBpeeringID", "", "Peering Id of nodeB")
nodeChost := *flag.String("nodeChost", "172.26.3.1", "host of nodeC (e.g. localhost)")
nodeChttpPort := *flag.String("nodeChttp", "9001", "port of nodeC (e.g. 9001)")
nodeCpeeringID := *flag.String("nodeCzmqID", "nodeC", "ZMQ Id of nodeC")
nodeChost := flag.String("nodeChost", "", "host of nodeC (e.g. localhost)")
nodeChttpPort := flag.String("nodeChttp", "", "port of nodeC (e.g. 9001)")
nodeCpeeringID := flag.String("nodeCpeeringID", "", "Peering Id of nodeC")
triggerNodeHost := *flag.String("triggerNodeHost", "172.26.5.1:3333", "host of trigger node (e.g. localhost:3333)")
triggerNodeWSHost := *flag.String("triggerNodeWSHost", "172.26.5.1:80", "host of trigger node web server (e.g. localhost:80)")
triggerNodeID := *flag.String("triggerNodeID", "triggernode", "Id of trigger node")
triggerNodeHost := flag.String("triggerNodeHost", "", "host of trigger node (e.g. localhost:3333)")
triggerNodeWSHost := flag.String("triggerNodeWSHost", "", "host of trigger node web server (e.g. localhost:80)")
triggerNodeID := flag.String("triggerNodeID", "", "Id of trigger node")
certFile := *flag.String("cert-file", "/cert/client.crt", "Certificate to talk to FReD")
keyFile := *flag.String("key-file", "/cert/client.key", "Keyfile to talk to FReD")
certFile := flag.String("cert-file", "", "Certificate to talk to FReD")
keyFile := flag.String("key-file", "", "Keyfile to talk to FReD")
littleCertFile := *flag.String("little-cert-file", "/cert/littleclient.crt", "Certificate to talk to FReD as \"littleclient\"")
littleKeyFile := *flag.String("little-key-file", "/cert/littleclient.key", "Keyfile to talk to FReD as \"littleclient\"")
littleCertFile := flag.String("little-cert-file", "", "Certificate to talk to FReD as \"littleclient\"")
littleKeyFile := flag.String("little-key-file", "", "Keyfile to talk to FReD as \"littleclient\"")
testRange := *flag.String("test-range", "-", "Give tests to execute as a dash-separated range. Omitted start or end become the lowest or highest possible index, respectively. Default: All tests (\"-\"). Examples: 2-7, 1-, -6, -")
testRange := flag.String("test-range", "-", "Give tests to execute as a dash-separated range. Omitted start or end become the lowest or highest possible index, respectively. Default: All tests (\"-\"). Examples: 2-7, 1-, -6, -")
flag.Parse()
port, _ := strconv.Atoi(nodeAhttpPort)
nodeA := grpcclient.NewNode(nodeAhost, port, certFile, keyFile)
port, _ = strconv.Atoi(nodeBhttpPort)
nodeB := grpcclient.NewNode(nodeBhost, port, certFile, keyFile)
port, _ = strconv.Atoi(nodeChttpPort)
nodeC := grpcclient.NewNode(nodeChost, port, certFile, keyFile)
port, _ = strconv.Atoi(nodeAhttpPort)
littleClient := grpcclient.NewNode(nodeAhost, port, littleCertFile, littleKeyFile)
port, _ := strconv.Atoi(*nodeAhttpPort)
nodeA := grpcclient.NewNode(*nodeAhost, port, *certFile, *keyFile)
port, _ = strconv.Atoi(*nodeBhttpPort)
nodeB := grpcclient.NewNode(*nodeBhost, port, *certFile, *keyFile)
port, _ = strconv.Atoi(*nodeChttpPort)
nodeC := grpcclient.NewNode(*nodeChost, port, *certFile, *keyFile)
port, _ = strconv.Atoi(*nodeAhttpPort)
littleClient := grpcclient.NewNode(*nodeAhost, port, *littleCertFile, *littleKeyFile)
time.Sleep(15 * time.Second)
config := &Config{
waitUser: waitUser,
waitUser: *waitUser,
nodeAhost: nodeAhost,
nodeAhttpPort: nodeAhttpPort,
nodeApeeringID: nodeApeeringID,
nodeAhost: *nodeAhost,
nodeAhttpPort: *nodeAhttpPort,
nodeApeeringID: *nodeApeeringID,
nodeBhost: nodeBhost,
nodeBhttpPort: nodeBhttpPort,
nodeBpeeringID: nodeBpeeringID,
nodeBhost: *nodeBhost,
nodeBhttpPort: *nodeBhttpPort,
nodeBpeeringID: *nodeBpeeringID,
nodeChost: nodeChost,
nodeChttpPort: nodeChttpPort,
nodeCpeeringID: nodeCpeeringID,
nodeChost: *nodeChost,
nodeChttpPort: *nodeChttpPort,
nodeCpeeringID: *nodeCpeeringID,
triggerNodeHost: triggerNodeHost,
triggerNodeWSHost: triggerNodeWSHost,
triggerNodeID: triggerNodeID,
triggerNodeHost: *triggerNodeHost,
triggerNodeWSHost: *triggerNodeWSHost,
triggerNodeID: *triggerNodeID,
certFile: certFile,
keyFile: keyFile,
certFile: *certFile,
keyFile: *keyFile,
littleCertFile: littleCertFile,
littleKeyFile: littleKeyFile,
littleCertFile: *littleCertFile,
littleKeyFile: *littleKeyFile,
nodeA: nodeA,
nodeB: nodeB,
nodeC: nodeC,
nodeA: nodeA,
nodeB: nodeB,
nodeC: nodeC,
littleClient: littleClient,
}
......@@ -116,7 +116,7 @@ func main() {
// parse testRange, starts at 1
minTest := 1
maxTest := len(testSuites)
testRangeSplit := strings.Split(testRange, "-")
testRangeSplit := strings.Split(*testRange, "-")
if len(testRangeSplit[0]) > 0 {
minTestInput, errMin := strconv.Atoi(testRangeSplit[0])
if errMin == nil {
......@@ -139,7 +139,7 @@ func main() {
// run tests
// minTest and maxTest are indexed with 1 at the beginning, but the slice starts at 0
for i := minTest-1; i < maxTest ; i++ {
for i := minTest - 1; i < maxTest; i++ {
testSuites[i].RunTests()
}
......@@ -153,9 +153,9 @@ func main() {
os.Exit(totalerrors)
}
func logNodeAction(node *grpcclient.Node, action string) {
func logNodeAction(node *grpcclient.Node, format string, a ...interface{}) {
wait()
log.Info().Str("node", node.Addr).Msg(action)
log.Info().Str("node", node.Addr).Msgf(format, a...)
}
func logNodeFailure(node *grpcclient.Node, expected, result string) {
......@@ -165,7 +165,7 @@ func logNodeFailure(node *grpcclient.Node, expected, result string) {
}
func wait() {
if waitUser {
if *waitUser {
log.Info().Msg("Please press enter to continue:")
_, _, _ = reader.ReadLine()
} else {
......
......@@ -11,14 +11,17 @@ type ReplicaSuite struct {
func (t *ReplicaSuite) RunTests() {
// Fun with replicas
logNodeAction(t.c.nodeA, "Adding nodeB as Replica node for KG1")
t.c.nodeA.AddKeygroupReplica("KG1", t.c.nodeBpeeringID, 0, false)
logNodeAction(t.c.nodeA, "Create keygroup KGRep")
t.c.nodeA.CreateKeygroup("KGRep", true, 0, false)
logNodeAction(t.c.nodeB, "Deleting the value from KG1")
t.c.nodeB.DeleteItem("KG1", "KG1Item", false)
logNodeAction(t.c.nodeA, "Adding nodeB as Replica node for KGRep")
t.c.nodeA.AddKeygroupReplica("KGRep", t.c.nodeBpeeringID, 0, false)
logNodeAction(t.c.nodeB, "Getting the deleted value in KG1")
_ = t.c.nodeB.GetItem("KG1", "KG1Item", true)
logNodeAction(t.c.nodeB, "Deleting the value from KGRep")
t.c.nodeB.DeleteItem("KGRep", "KGRepItem", false)
logNodeAction(t.c.nodeB, "Getting the deleted value in KGRep")
_ = t.c.nodeB.GetItem("KGRep", "KGRepItem", true)
// Test sending data between nodes
logNodeAction(t.c.nodeB, "Creating a new Keygroup (KGN) in nodeB, setting nodeA as Replica node")
......
......@@ -7,12 +7,35 @@ services:
dockerfile: tests/3NodeTest/Dockerfile
image: fred/tester:local
container_name: tester
command: --nodeAhost=172.26.1.1 --nodeAhttp=9001 --nodeBhost=172.26.2.1 --nodeBhttp=9001 --nodeChost=172.26.3.1 --nodeChttp=9001 --cert-file "/cert/client.crt" --key-file "/cert/client.key" --little-cert-file "/cert/littleclient.crt" --little-key-file "/cert/littleclient.key" --wait-user=false
command: "--nodeAhost=172.26.1.1 \
--nodeAhttp=9001 \
--nodeApeeringID=nodeA \
--nodeBhost=172.26.2.1 \
--nodeBhttp=9001 \
--nodeBpeeringID=nodeB \
--nodeChost=172.26.3.1 \
--nodeChttp=9001 \
--nodeCpeeringID=nodeC \
--triggerNodeHost=172.26.5.1:3333 \
--triggerNodeWSHost=172.26.5.1:80 \
--triggerNodeID=triggernode \
--cert-file='/cert/client.crt' \
--key-file='/cert/client.key' \
--little-cert-file='/cert/littleclient.crt' \
--little-key-file='/cert/littleclient.key' \
--test-range=${TEST_RANGE} \
--wait-user=false"
volumes:
- ../../nase/tls/client.crt:/cert/client.crt
- ../../nase/tls/client.key:/cert/client.key
- ../../nase/tls/littleclient.crt:/cert/littleclient.crt
- ../../nase/tls/littleclient.key:/cert/littleclient.key
depends_on:
- nodeA-1
- nodeA-2
- nodeA-3
- nodeB
- nodeC
networks:
fredwork:
ipv4_address: 172.26.4.1
......
.PHONY: test clean
.PHONY: all clean 3nodetest 3nodetest-debug-nodeB 3n-all 3n-standard 3n-replica 3n-trigger 3n-immutable 3n-expiry 3n-selfreplica 3n-auth fred failtest alexandra
all: clean 3n-all clean
clean: ## clean up all resources
@docker-compose -f etcd.yml -f nodeA.yml -f nodeB.yml -f nodeC.yml -f 3NodeTester.yml -f trigger.yml down --remove-orphan
@docker network remove fredwork
3nodetest: ## start all containers with docker compose and run the test dockerized
3n-all: export TEST_RANGE = -
3n-standard: export TEST_RANGE = 1-1
3n-replica: export TEST_RANGE = 2-2
3n-trigger: export TEST_RANGE = 3-3
3n-immutable: export TEST_RANGE = 4-4
3n-expiry: export TEST_RANGE = 5-5
3n-selfreplica: export TEST_RANGE = 6-6
3n-auth: export TEST_RANGE = 7-7
3n-all 3n-standard 3n-replica 3n-trigger 3n-immutable 3n-expiry 3n-selfreplica 3n-auth: 3nodetest
3nodetest: clean ## start all containers with docker compose and run the test dockerized
@docker network create fredwork --gateway 172.26.0.1 --subnet 172.26.0.0/16 || true
@docker-compose -f etcd.yml -f nodeA.yml -f nodeB.yml -f nodeC.yml -f 3NodeTester.yml -f trigger.yml build
@docker-compose -f etcd.yml -f nodeA.yml -f nodeB.yml -f nodeC.yml -f 3NodeTester.yml -f trigger.yml up --force-recreate --abort-on-container-exit --renew-anon-volumes --remove-orphans
3nodetest-debug-nodeB: export TEST_RANGE = -
3nodetest-debug-nodeB: ## same as "test" but with nodeB in debugger mode
@docker network create fredwork --gateway 172.26.0.1 --subnet 172.26.0.0/16 || true
@docker-compose -f etcd.yml -f nodeA.yml -f nodeB-debug.yml -f nodeC.yml -f 3NodeTester.yml -f trigger.yml build
@docker-compose -f etcd.yml -f nodeA.yml -f nodeB-debug.yml -f nodeC.yml -f 3NodeTester.yml -f trigger.yml up --force-recreate --abort-on-container-exit --renew-anon-volumes --remove-orphans
fred: ## Start all containers except the tester so that other clients can access them
fred: clean ## Start all containers except the tester so that other clients can access them
@docker network create fredwork --gateway 172.26.0.1 --subnet 172.26.0.0/16 || true
@docker-compose -f etcd.yml -f nodeA.yml -f nodeB.yml -f nodeC.yml -f trigger.yml build
@docker-compose -f etcd.yml -f nodeA.yml -f nodeB.yml -f nodeC.yml -f trigger.yml up --force-recreate --renew-anon-volumes --remove-orphans
......@@ -23,7 +36,7 @@ failtest: ## Start the failtest
@docker-compose -f failtest.yml build
@docker-compose -f failtest.yml up --force-recreate --abort-on-container-exit --renew-anon-volumes
alexandra:
alexandra: clean ## start fred with alexandra
@docker network create fredwork --gateway 172.26.0.1 --subnet 172.26.0.0/16 || true
@docker-compose -f etcd.yml -f nodeA.yml -f nodeB.yml -f nodeC.yml -f trigger.yml -f alexandra.yml build
@docker-compose -f etcd.yml -f nodeA.yml -f nodeB.yml -f nodeC.yml -f trigger.yml -f alexandra.yml up --abort-on-container-exit --force-recreate --renew-anon-volumes --remove-orphans
......@@ -12,7 +12,12 @@ services:
dockerfile: alexandra.Dockerfile
image: fred/alexandra:local
container_name: alexandra
command: --lighthouse 172.26.2.1:9001 --ca-cert /cert/ca.crt --alexandra-key /cert/alexandra.key --alexandra-cert /cert/alexandra.crt --clients-key /cert/alexandra.key --clients-cert /cert/alexandra.crt
command: "--lighthouse 172.26.2.1:9001 \
--ca-cert /cert/ca.crt \
--alexandra-key /cert/alexandra.key \
--alexandra-cert /cert/alexandra.crt \
--clients-key /cert/alexandra.key \
--clients-cert /cert/alexandra.crt"
volumes:
- ../../nase/tls/alexandra.crt:/cert/alexandra.crt
- ../../nase/tls/alexandra.key:/cert/alexandra.key
......
......@@ -3,6 +3,10 @@ version: "3.7"
services:
# this haproxy proxies between 172.26.1.101, 172.26.1.102, 172.26.1.103
nodeAproxy:
depends_on:
- nodeA-1
- nodeA-2
- nodeA-3
build:
context: .
dockerfile: haproxy.Dockerfile
......@@ -18,7 +22,7 @@ services:
nodeA-1:
depends_on:
- nodeAproxy
- etcd
- storeA
build: ../../
image: fred/fred:local
......@@ -56,7 +60,7 @@ services:
nodeA-2:
depends_on:
- nodeAproxy
- etcd
- storeA
build: ../../
image: fred/fred:local
......@@ -94,7 +98,7 @@ services:
nodeA-3:
depends_on:
- nodeAproxy
- etcd
- storeA
build: ../../
image: fred/fred:local
......
......@@ -6,6 +6,7 @@ services:
context: ../../
dockerfile: debug.Dockerfile
depends_on:
- etcd
- storeB
image: fred/fred:local-debug
security_opt:
......
......@@ -4,6 +4,7 @@ services:
nodeB:
build: ../../
depends_on:
- etcd
- storeB
image: fred/fred:local
container_name: nodeB
......
......@@ -4,6 +4,7 @@ services:
nodeC:
build: ../../
depends_on:
- etcd
- storeC
image: fred/fred:local
container_name: nodeC
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment