Commit 41067e37 authored by pfandzelter's avatar pfandzelter
Browse files

clean up a bit

parent a796fcf8
Pipeline #7471 passed with stages
in 17 minutes and 54 seconds
# FReD Specific
# terraform secrets
secret.tfvars
# fred binary
frednode
......@@ -17,8 +14,8 @@ test.db/
.vscode/
# Created by https://www.toptal.com/developers/gitignore/api/go,linux,macos,windows,microsoftoffice,terraform,node,latex,intellij
# Edit at https://www.toptal.com/developers/gitignore?templates=go,linux,macos,windows,microsoftoffice,terraform,node,latex,intellij
# Created by https://www.toptal.com/developers/gitignore/api/go,linux,macos,windows,microsoftoffice,node,latex,intellij
# Edit at https://www.toptal.com/developers/gitignore?templates=go,linux,macos,windows,microsoftoffice,node,latex,intellij
### Go ###
# Binaries for programs and plugins
......@@ -502,35 +499,6 @@ dist
# Stores VSCode versions used for testing VSCode extensions
.vscode-test
### Terraform ###
# Local .terraform directories
**/.terraform/*
# .tfstate files
*.tfstate
*.tfstate.*
# Crash log files
crash.log
# Ignore any .tfvars files that are generated automatically for each Terraform run. Most
# .tfvars files are managed as part of configuration and so should be included in
# version control.
#
# example.tfvars
# Ignore override files as they are usually used to override resources locally and so
# are not checked in
override.tf
override.tf.json
*_override.tf
*_override.tf.json
# Include override files you do wish to add to version control using negated pattern
# !example_override.tf
# Include tfplan files to ignore the plan output of command: terraform plan -out=tfplan
# example: *tfplan*
### Windows ###
# Windows thumbnail cache files
......@@ -661,4 +629,4 @@ fabric.properties
# https://plugins.jetbrains.com/plugin/12206-codestream
.idea/codestream.xml
# End of https://www.toptal.com/developers/gitignore/api/go,linux,macos,windows,microsoftoffice,terraform,node,latex,intellij
\ No newline at end of file
# End of https://www.toptal.com/developers/gitignore/api/go,linux,macos,windows,microsoftoffice,node,latex,intellij
\ No newline at end of file
......@@ -70,15 +70,3 @@ container:
- docker login -u $CI_REGISTRY_USER -p $CI_REGISTRY_PASSWORD $CI_REGISTRY
- docker tag git.tu-berlin.de:5000/mcc-fred/fred/fred git.tu-berlin.de:5000/mcc-fred/fred/fred:$CONTAINER_TAG
- docker push git.tu-berlin.de:5000/mcc-fred/fred/fred:$CONTAINER_TAG
# this adds a new stage to the CI process that tests the terraform files
#terraform:
# stage: test_aws
# script:
# - cd terraform/small-test
# - TF_IN_AUTOMATION=true
# - CONTAINER_TAG=${components[0]}
# - if [ "$CI_COMMIT_REF_NAME" == "master" ]; then CONTAINER_TAG=latest; fi;
# - terraform init -input=false
# - terraform plan -input=false -var="aws_access_key=$AWS_ACCESS_KEY" -var="aws_secret_key=$AWS_SECRET_KEY" -var="gitlab_repo_username=$CI_REGISTRY_USER" -var="gitlab_repo_password=$CI_REGISTRY_PASSWORD" -var="identifier=$CONTAINER_TAG"
# building the binary
FROM golang:1.15-alpine as golang
MAINTAINER Tobias Pfandzelter <tp@mcc.tu-berlin.de>
LABEL maintainer="tp@mcc.tu-berlin.de"
WORKDIR /go/src/gitlab.tu-berlin.de/mcc-fred/fred/
......@@ -27,10 +27,10 @@ FROM scratch
WORKDIR /
COPY --from=golang /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/
COPY --from=golang /go/bin/frednode frednode
COPY --from=golang /go/bin/frednode fred
EXPOSE 443
EXPOSE 5555
ENTRYPOINT ["./frednode"]
ENV PATH=.
CMD ["fred"]
\ No newline at end of file
......@@ -10,25 +10,23 @@ For development, it is recommended to install [GoLand](https://www.jetbrains.com
It is probably easiest to use the dockerfiles to run Fred. An example setup can be found in `tests/3NodeTest`
To use Terraform, install [Terraform](https://www.terraform.io/downloads.html).
## Git Workflow
Setup git environment with `sh ./env-setup.sh` (installs git hooks). Be sure to have go installed...
Setup git environment with `sh ./ci/env-setup.sh` (installs git hooks). Be sure to have go installed...
The `master` branch is protected and only approved pull requests can push to it. Most important part of
the workflow is `rebase`, heres a refresher on merging vs rebasing https://www.atlassian.com/git/tutorials/merging-vs-rebasing.
the workflow is `rebase`, [here's](https://www.atlassian.com/git/tutorials/merging-vs-rebasing) a refresher on merging vs rebasing.
How do I push changes to the `master` branch?
1. Switch to `master` -> `git checkout master`
2. Update `master` -> `git pull --rebase` (ALWAYS use `rebase` when pulling!!!)
3. Create new branch from `master` -> `git checkout -b tp/new-feature` (where 'tp/' is your own name/abbreviation)
4. Work on branch and push changes
5. Rebase master onto branch to not have merge conflicts later -> `git pull origin master --rebase` (AGAIN use`--rebase`)
6. Push branch again, this time force push to include rebased master (`git push --force`)
7. Create a pull request from gitlab.tu-berlin.de
8. Get pull request reviewed and merge it into master
1. Switch to `master` -> `git checkout master`
2. Update `master` -> `git pull --rebase` (ALWAYS use `rebase` when pulling!!!)
3. Create new branch from `master` -> `git checkout -b tp/new-feature` (where 'tp/' is your own name/abbreviation)
4. Work on branch and push changes
5. Rebase master onto branch to not have merge conflicts later -> `git pull origin master --rebase` (AGAIN use`--rebase`)
6. Push branch again, this time force push to include rebased master (`git push --force`)
7. Create a pull request from gitlab.tu-berlin.de
8. Get pull request reviewed and merge it into master
Some last words, keep pull requests small (not 100 files changed etc :D), so they are easier to review and rather create a lot of pull requests than one big
......@@ -58,12 +56,12 @@ To delete the table:
AWS_PAGER="" aws dynamodb delete-table --table-name fred
```
# Tips and Tricks
## Tips and Tricks
- To write a new client a language binding from the proto file (`proto/client/client.proto`) needs to be created. The two tests (in `tests/`) use Go and Python and can be used as inspiration.
- FReD requires the use of certificates to check whether a node has access to a specific keygroup or not.
If access control is not necessary every node can just use the same certificate => they all have access to all the data. (The nodes in the 3NodeTest all use the same certificates, for example)
# Caching in Nameservice
## Caching in Nameservice
A CLI flag has been added to optionally enable caching for the nameservice. Pass `--nase-cached` to activate caching. Default is off.
......@@ -52,19 +52,6 @@ RUN set -eux; \
\
docker --version
# # Install terraform
# ENV TERRAFORM_VER 0.12.9
# RUN wget -O terraform.zip "https://releases.hashicorp.com/terraform/${TERRAFORM_VER}/terraform_${TERRAFORM_VER}_linux_amd64.zip"
# RUN apt-get update && apt-get install -y --no-install-recommends \
# unzip \
# && apt-get clean \
# && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/*
# RUN unzip terraform.zip
# RUN mv terraform /usr/local/bin/
# Install Docker Compose
RUN curl -L "https://github.com/docker/compose/releases/download/1.26.2/docker-compose-$(uname -s)-$(uname -m)" -o /usr/local/bin/docker-compose
RUN chmod +x /usr/local/bin/docker-compose
\ No newline at end of file
cp ./tools/pre-commit ../.git/hooks/pre-commit
chmod +x ../.git/hooks/pre-commit
\ No newline at end of file
# FReD Node Configuration File
[location]
# latitude of the server
lat = 52.514927933123914
# longitude of the server
lng = 13.32676300345363
[server]
host = ":9001"
ssl = false
[storage]
# adaptor can be either "memory" or "badgerdb" or "remote"
adaptor = "badgerdb"
[peering]
host = ":5555"
[dynamodb]
table = "fredtable"
region = "eu-central-1"
[remotestore]
host = "localhost:1337"
[log]
# debug,info,warn,error,fatal,panic
level = "debug"
# dev: print colored output to stdout
# prod: print json to stdout
handler = "dev"
[nase]
# adress of etcd server
host = "172.26.1.1:2379"
\ No newline at end of file
......@@ -2,7 +2,8 @@
# building the binary
FROM golang:1.15-buster as golang
MAINTAINER Tobias Pfandzelter <tp@mcc.tu-berlin.de>
LABEL maintainer="tp@mcc.tu-berlin.de"
WORKDIR /go/src/gitlab.tu-berlin.de/mcc-fred/fred/
RUN apt update && apt install -y ca-certificates git && rm -rf /var/cache/apk/*
......
cp ./ci/tools/pre-commit ./.git/hooks/pre-commit
chmod +x ./.git/hooks/pre-commit
\ No newline at end of file
......@@ -2,7 +2,7 @@
# building the binary
FROM golang:1.15-alpine as golang
MAINTAINER Tobias Pfandzelter <tp@mcc.tu-berlin.de>
LABEL maintainer="tp@mcc.tu-berlin.de"
WORKDIR /go/src/gitlab.tu-berlin.de/mcc-fred/fred/
......@@ -31,4 +31,5 @@ COPY --from=golang /go/bin/storageserver storageserver
EXPOSE 1337
ENTRYPOINT ["./storageserver"]
\ No newline at end of file
ENV PATH=.
ENTRYPOINT ["storageserver"]
\ No newline at end of file
data "aws_ami" "amazonlinux2" {
most_recent = true
filter {
name = "name"
values = ["amzn2-ami-hvm-2.0.*-ebs"]
}
filter {
name = "architecture"
values = ["x86_64"]
}
filter {
name = "virtualization-type"
values = ["hvm"]
}
owners = ["137112412989"] # Canonical
}
\ No newline at end of file
# FReD Node Configuration File
[location]
# latitude of the server
lat = 52.514927933123914
# longitude of the server
lng = 13.32676300345363
[webserver]
# host = "localhost"
port = 443
ssl = true
[storage]
# adaptor can be either "leveldb" or "memory"
adaptor = "leveldb"
[peering]
port = 5555
[leveldb]
path = "./db"
[log]
# debug,info,warn,error,fatal,panic
level = "debug"
# dev: print colored output to stdout
# prod: print json to stdout
handler = "dev"
\ No newline at end of file
resource "aws_route53_record" "dns_record" {
# Use the ID of the Hosted Zone we retrieved earlier
zone_id = data.aws_route53_zone.hosted_zone.zone_id
# Set the name of the record, e.g. pc.mydomain.com
name = "${count.index}.${var.domain_name}"
count = var.instance_count
# We're pointing to an IP address so we need to use an A record
type = "A"
# We'll set the TTL of the record to 30 minutes (1800 seconds)
ttl = "1800"
records = [ aws_eip.test-eip[count.index].public_ip ]
}
\ No newline at end of file
data "aws_route53_zone" "hosted_zone" {
name = var.domain_hosted_zone
}
\ No newline at end of file
resource "aws_eip" "test-eip" {
count = var.instance_count
}
\ No newline at end of file
resource "aws_eip_association" "test-eip-assoc" {
count = var.instance_count
allocation_id = aws_eip.test-eip[count.index].id
instance_id = aws_instance.fred_instance[count.index].id
}
\ No newline at end of file
resource "aws_key_pair" "keypair" {
key_name = var.key_name
public_key = var.key_pub
}
\ No newline at end of file
resource "aws_instance" "fred_instance" {
ami = data.aws_ami.amazonlinux2.id
instance_type = var.instance_type
key_name = aws_key_pair.keypair.key_name
count = var.instance_count
security_groups = var.security_groups
provisioner "file" {
source = "./fred-node/config.toml"
destination = "/tmp/config.toml"
}
provisioner "file" {
source = "./fred-node/setup_node.sh"
destination = "/tmp/script.sh"
}
provisioner "remote-exec" {
inline = [
"chmod +x /tmp/script.sh",
# Args:
# $1: username to access gitlab registry
# $2: password to access gitlab registry
# $3: identifier to use the correct Docker container
# $4: "host" argument for FReD, in this case the domain name returned from R53
# $5: flags that should be passed to FReD
"/tmp/script.sh ${var.gitlab_repo_username} ${var.gitlab_repo_password} ${var.identifier} ${aws_route53_record.dns_record[count.index].fqdn} ${var.fred_flags[count.index]}",
]
}
connection {
type = "ssh"
user = "ec2-user"
private_key = var.key_prv
host = self.public_ip
}
tags = {
Name = "${var.name}-${count.index}"
type = "fred"
}
}
\ No newline at end of file
output "server-ip" {
value = [ aws_eip_association.test-eip-assoc.*.public_ip ]
}
output "server-domain-name" {
value = [ aws_route53_record.dns_record.*.fqdn ]
}
#!/usr/bin/env bash
sudo yum update -y --quiet
sudo yum install docker -y --quiet
echo "$2" > ./gitlabtoken
sudo systemctl start docker
sudo cat ./gitlabtoken | sudo docker login -u="$1" --password-stdin git.tu-berlin.de:5000
sudo rm gitlabtoken
sudo docker pull git.tu-berlin.de:5000/mcc-fred/fred/fred:"$3"
sudo docker run -it \
--name=fred \
-d \
--restart=unless-stopped \
-p 443:443 \
-p 80:80 \
-p 5555:5555 \
-v /tmp/config.toml:/config.toml \
git.tu-berlin.de:5000/mcc-fred/fred/fred:"$3" --config config.toml --ws-host "$4" "$5"
\ No newline at end of file
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment