From e6ae13812c68798eb74288a6942c89d1e779799e Mon Sep 17 00:00:00 2001 From: Damien Garros Date: Tue, 21 Mar 2017 14:52:32 -0700 Subject: [PATCH 1/6] Initial Commit --- docker-compose/.env | 1 + docker-compose/Makefile | 14 +++ docker-compose/README.md | 0 docker-compose/configs/toddagent.toml | 20 +++++ docker-compose/configs/toddserver.toml | 51 +++++++++++ docker-compose/docker-compose.yml | 105 ++++++++++++++++++++++ docker-compose/objects/.bash_history | 4 + docker-compose/objects/group-dc.yaml | 7 ++ docker-compose/objects/group-hq.yaml | 7 ++ docker-compose/objects/test-http.yaml | 13 +++ docker-compose/objects/test-iperf.yaml | 14 +++ docker-compose/objects/test-ping-2.yaml | 14 +++ docker-compose/objects/test-ping-dns.yaml | 16 ++++ 13 files changed, 266 insertions(+) create mode 100644 docker-compose/.env create mode 100644 docker-compose/Makefile create mode 100644 docker-compose/README.md create mode 100644 docker-compose/configs/toddagent.toml create mode 100644 docker-compose/configs/toddserver.toml create mode 100644 docker-compose/docker-compose.yml create mode 100644 docker-compose/objects/.bash_history create mode 100644 docker-compose/objects/group-dc.yaml create mode 100644 docker-compose/objects/group-hq.yaml create mode 100644 docker-compose/objects/test-http.yaml create mode 100644 docker-compose/objects/test-iperf.yaml create mode 100644 docker-compose/objects/test-ping-2.yaml create mode 100644 docker-compose/objects/test-ping-dns.yaml diff --git a/docker-compose/.env b/docker-compose/.env new file mode 100644 index 0000000..357822c --- /dev/null +++ b/docker-compose/.env @@ -0,0 +1 @@ +HOST_IP=192.168.0.1 diff --git a/docker-compose/Makefile b/docker-compose/Makefile new file mode 100644 index 0000000..823c4c4 --- /dev/null +++ b/docker-compose/Makefile @@ -0,0 +1,14 @@ + +# Need to start Toddserver twice because it crashs if RabbitMq is not ready +start: + docker-compose up -d + docker-compose up -d toddserver + +stop: + docker-compose down + +agents: + docker exec -it dockercompose_toddserver_1 todd agents + +cli: + docker exec -it dockercompose_toddserver_1 bash diff --git a/docker-compose/README.md b/docker-compose/README.md new file mode 100644 index 0000000..e69de29 diff --git a/docker-compose/configs/toddagent.toml b/docker-compose/configs/toddagent.toml new file mode 100644 index 0000000..9bbad9a --- /dev/null +++ b/docker-compose/configs/toddagent.toml @@ -0,0 +1,20 @@ +# Describes parameters for the "comms" system, which manages communications between +# the server and the agents +[Comms] +User = guest # Username for comms +Password = guest # Password for comms +Host = rabbitmq # Hostname or IP address for comms +Port = 5672 # Port for comms +Plugin = rabbitmq # Comms plugin to use (i.e. "rabbitmq") + +# Describes parameters for local resources, such as network or filesystem resources +[LocalResources] +DefaultInterface = eth0 # Dictates what network interface is used for testing + # purposes (i.e. informs the todd-server which IP + # address can be used + +# IPAddrOverride = 192.168.99.100 # Overrides DefaultInterface by providing a specific IP + # address rather + +OptDir = /opt/todd/agent # Operational directory for the agent. Houses things like + # cache files, user-defined testlets, etc. diff --git a/docker-compose/configs/toddserver.toml b/docker-compose/configs/toddserver.toml new file mode 100644 index 0000000..c385b3b --- /dev/null +++ b/docker-compose/configs/toddserver.toml @@ -0,0 +1,51 @@ +# ToDD's API +[API] +Host = 0.0.0.0 +Port = 8080 + +# Serves assets like collectors, testlets, etc. +[Assets] +IP = 0.0.0.0 +Port = 8090 + +# Describes parameters for the "comms" system, which manages communications between +# the server and the agents +[Comms] +User = guest # Username for comms +Password = guest # Password for comms +Host = rabbitmq # Hostname or IP address for comms +Port = 5672 # Port for comms +Plugin = rabbitmq # Comms plugin to use (i.e. "rabbitmq") + +# Parameters for database connectivity +[DB] +Host = etcd # Hostname or IP address for database +Port = 4001 # Port for database +Plugin = etcd # Database plugin to use (i.e. "etcd") + +# Parameters for time-series database connectivity +[TSDB] +Host = influxdb # Hostname or IP address for tsdb +Port = 8086 # Port for tsdb +Plugin = influxdb # TSDB plugin to use (i.e. "influxdb") +DatabaseName = todd # Todd + +[Grouping] +Interval = 10 # Interval (in seconds) for the grouping calculation + # to run on the server + +[Testing] +Timeout = 30 # This is the timer (in seconds) that a test will be + # allowed to live + +# Describes parameters for local resources, such as network or filesystem resources +[LocalResources] +DefaultInterface = eth2 # Dictates what network interface is used for testing + # purposes (i.e. informs the todd-server which IP + # address can be used + +IPAddrOverride = toddserver # Overrides DefaultInterface by providing a specific IP + # address rather + +OptDir = /opt/todd/agent # Operational directory for the agent. Houses things like + # cache files, user-defined testlets, etc. diff --git a/docker-compose/docker-compose.yml b/docker-compose/docker-compose.yml new file mode 100644 index 0000000..ed092e0 --- /dev/null +++ b/docker-compose/docker-compose.yml @@ -0,0 +1,105 @@ +version: '2' + +services: +# ------------------------------------------------------------------------- +# Todd Server +# ------------------------------------------------------------------------- + toddserver: + image: toddproject/todd + command: + - 'todd-server' + volumes: + - /etc/localtime:/etc/localtime + - ./objects:/root + - ./configs/toddserver.toml:/etc/todd/server.cfg + ports: + - "8080:8080" + depends_on: + - etcd + - rabbitmq + - influxdb + networks: + - default + - agent-hq + - agent-dc + +# ------------------------------------------------------------------------- +# Todd Agent - Group DC +# ------------------------------------------------------------------------- + agent-dc: + image: toddproject/todd + command: + - 'todd-agent' + volumes: + - /etc/localtime:/etc/localtime + - ./configs/toddagent.toml:/etc/todd/agent.cfg + networks: + - agent-dc + +# ------------------------------------------------------------------------- +# Todd Agent - Group HQ +# ------------------------------------------------------------------------- + agent-hq: + image: toddproject/todd + volumes: + - /etc/localtime:/etc/localtime + - ./configs/toddagent.toml:/etc/todd/agent.cfg + command: + - 'todd-agent' + networks: + - agent-hq + +# ------------------------------------------------------------------------- +# Etcd +# ------------------------------------------------------------------------- + etcd: + image: quay.io/coreos/etcd:v3.1.3 + ports: + - '2379:2379' + - '2380:2380' + - '4001:4001' + command: /usr/local/bin/etcd -name etcd0 -advertise-client-urls http://${HOST_IP}:2379,http://${HOST_IP}:4001 -listen-client-urls http://0.0.0.0:2379,http://0.0.0.0:4001 -initial-advertise-peer-urls http://${HOST_IP}:2380 -listen-peer-urls http://0.0.0.0:2380 -initial-cluster-token etcd-cluster-1 -initial-cluster etcd0=http://${HOST_IP}:2380 -initial-cluster-state new + volumes: + - /etc/localtime:/etc/localtime + +# ------------------------------------------------------------------------- +# Rabbitmq +# ------------------------------------------------------------------------- + rabbitmq: + image: rabbitmq:3 + volumes: + - /etc/localtime:/etc/localtime + ports: + - "5672:5672" + networks: + - default + - agent-hq + - agent-dc + +# ------------------------------------------------------------------------- +# Influxdb +# ------------------------------------------------------------------------- + influxdb: + image: influxdb:1.1.1-alpine + volumes: + - /etc/localtime:/etc/localtime + environment: + - "INFLUXDB_ADMIN_ENABLED=true" + ports: + - "8083:8083" + - "8086:8086" + networks: + - default + +# ------------------------------------------------------------------------- +# Network Configuration +# ------------------------------------------------------------------------- +networks: + agent-dc: + ipam: + config: + - subnet: 172.25.1.0/24 + agent-hq: + ipam: + config: + - subnet: 172.25.2.0/24 diff --git a/docker-compose/objects/.bash_history b/docker-compose/objects/.bash_history new file mode 100644 index 0000000..0d5499d --- /dev/null +++ b/docker-compose/objects/.bash_history @@ -0,0 +1,4 @@ +todd agents +make agents +make agents +exit diff --git a/docker-compose/objects/group-dc.yaml b/docker-compose/objects/group-dc.yaml new file mode 100644 index 0000000..0867686 --- /dev/null +++ b/docker-compose/objects/group-dc.yaml @@ -0,0 +1,7 @@ +--- +type: group +label: agent-dc +spec: + group: dc + matches: + - within_subnet: "172.25.1.0/24" diff --git a/docker-compose/objects/group-hq.yaml b/docker-compose/objects/group-hq.yaml new file mode 100644 index 0000000..8d99388 --- /dev/null +++ b/docker-compose/objects/group-hq.yaml @@ -0,0 +1,7 @@ +--- +type: group +label: agent-hq +spec: + group: hq + matches: + - within_subnet: "172.25.2.0/24" diff --git a/docker-compose/objects/test-http.yaml b/docker-compose/objects/test-http.yaml new file mode 100644 index 0000000..fc513fb --- /dev/null +++ b/docker-compose/objects/test-http.yaml @@ -0,0 +1,13 @@ +--- +# Example test file +type: testrun +label: test-http +spec: + targettype: uncontrolled + source: + name: docker + app: http + args: "" + target: + - salesforce.com + - portal.office.com diff --git a/docker-compose/objects/test-iperf.yaml b/docker-compose/objects/test-iperf.yaml new file mode 100644 index 0000000..2e90f26 --- /dev/null +++ b/docker-compose/objects/test-iperf.yaml @@ -0,0 +1,14 @@ +--- +# Example test file +type: testrun +label: test-iperf +spec: + targettype: group + source: + name: agent-dc + app: iperf + args: "-c {{ target }}" + target: + name: agent-hq + app: iperf + args: "-s" diff --git a/docker-compose/objects/test-ping-2.yaml b/docker-compose/objects/test-ping-2.yaml new file mode 100644 index 0000000..c273bea --- /dev/null +++ b/docker-compose/objects/test-ping-2.yaml @@ -0,0 +1,14 @@ +--- +# Example test file +type: testrun +label: test-ping +spec: + targettype: group + source: + name: agent-dc + app: ping + args: "-c 10" + target: + name: agent-hq + app: ping + args: "-c 10" diff --git a/docker-compose/objects/test-ping-dns.yaml b/docker-compose/objects/test-ping-dns.yaml new file mode 100644 index 0000000..b7481af --- /dev/null +++ b/docker-compose/objects/test-ping-dns.yaml @@ -0,0 +1,16 @@ +--- +# Example test file +type: testrun +label: test-ping-dns-dc +spec: + targettype: uncontrolled + target: + name: agent-dc + app: ping + args: "-c 10" + source: + name: agent-dc + + target: + - 4.2.2.2 + - 8.8.8.8 From 59fab7ae70a6065f5072eb9002c3f6f5b71f9ab5 Mon Sep 17 00:00:00 2001 From: Damien Garros Date: Tue, 21 Mar 2017 15:07:44 -0700 Subject: [PATCH 2/6] Remove old tests --- docker-compose/objects/test-iperf.yaml | 14 -------------- docker-compose/objects/test-ping-2.yaml | 14 -------------- 2 files changed, 28 deletions(-) delete mode 100644 docker-compose/objects/test-iperf.yaml delete mode 100644 docker-compose/objects/test-ping-2.yaml diff --git a/docker-compose/objects/test-iperf.yaml b/docker-compose/objects/test-iperf.yaml deleted file mode 100644 index 2e90f26..0000000 --- a/docker-compose/objects/test-iperf.yaml +++ /dev/null @@ -1,14 +0,0 @@ ---- -# Example test file -type: testrun -label: test-iperf -spec: - targettype: group - source: - name: agent-dc - app: iperf - args: "-c {{ target }}" - target: - name: agent-hq - app: iperf - args: "-s" diff --git a/docker-compose/objects/test-ping-2.yaml b/docker-compose/objects/test-ping-2.yaml deleted file mode 100644 index c273bea..0000000 --- a/docker-compose/objects/test-ping-2.yaml +++ /dev/null @@ -1,14 +0,0 @@ ---- -# Example test file -type: testrun -label: test-ping -spec: - targettype: group - source: - name: agent-dc - app: ping - args: "-c 10" - target: - name: agent-hq - app: ping - args: "-c 10" From 09775ece46b06b56b448f67f08696472a3cf55e1 Mon Sep 17 00:00:00 2001 From: Damien Garros Date: Tue, 21 Mar 2017 15:08:06 -0700 Subject: [PATCH 3/6] Remove external ports for now --- docker-compose/docker-compose.yml | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/docker-compose/docker-compose.yml b/docker-compose/docker-compose.yml index ed092e0..5b0651f 100644 --- a/docker-compose/docker-compose.yml +++ b/docker-compose/docker-compose.yml @@ -12,8 +12,8 @@ services: - /etc/localtime:/etc/localtime - ./objects:/root - ./configs/toddserver.toml:/etc/todd/server.cfg - ports: - - "8080:8080" + # ports: + # - "8080:8080" depends_on: - etcd - rabbitmq @@ -54,10 +54,10 @@ services: # ------------------------------------------------------------------------- etcd: image: quay.io/coreos/etcd:v3.1.3 - ports: - - '2379:2379' - - '2380:2380' - - '4001:4001' + # ports: + # - '2379:2379' + # - '2380:2380' + # - '4001:4001' command: /usr/local/bin/etcd -name etcd0 -advertise-client-urls http://${HOST_IP}:2379,http://${HOST_IP}:4001 -listen-client-urls http://0.0.0.0:2379,http://0.0.0.0:4001 -initial-advertise-peer-urls http://${HOST_IP}:2380 -listen-peer-urls http://0.0.0.0:2380 -initial-cluster-token etcd-cluster-1 -initial-cluster etcd0=http://${HOST_IP}:2380 -initial-cluster-state new volumes: - /etc/localtime:/etc/localtime @@ -69,8 +69,8 @@ services: image: rabbitmq:3 volumes: - /etc/localtime:/etc/localtime - ports: - - "5672:5672" + # ports: + # - "5672:5672" networks: - default - agent-hq @@ -85,9 +85,9 @@ services: - /etc/localtime:/etc/localtime environment: - "INFLUXDB_ADMIN_ENABLED=true" - ports: - - "8083:8083" - - "8086:8086" + # ports: + # - "8083:8083" + # - "8086:8086" networks: - default From 8578a55733ada85d760b0f0a3ccfbf9955968084 Mon Sep 17 00:00:00 2001 From: Damien Garros Date: Tue, 21 Mar 2017 15:08:34 -0700 Subject: [PATCH 4/6] Delete bash history --- docker-compose/objects/.bash_history | 4 ---- 1 file changed, 4 deletions(-) delete mode 100644 docker-compose/objects/.bash_history diff --git a/docker-compose/objects/.bash_history b/docker-compose/objects/.bash_history deleted file mode 100644 index 0d5499d..0000000 --- a/docker-compose/objects/.bash_history +++ /dev/null @@ -1,4 +0,0 @@ -todd agents -make agents -make agents -exit From 5fb78a87b2808d7a4da065bdff186813592674ef Mon Sep 17 00:00:00 2001 From: Damien Garros Date: Tue, 21 Mar 2017 15:09:10 -0700 Subject: [PATCH 5/6] Add basic instructions in README --- docker-compose/Makefile | 18 ++++++++++-- docker-compose/README.md | 35 +++++++++++++++++++++++ docker-compose/objects/test-http.yaml | 2 +- docker-compose/objects/test-ping-dns.yaml | 5 +--- 4 files changed, 52 insertions(+), 8 deletions(-) diff --git a/docker-compose/Makefile b/docker-compose/Makefile index 823c4c4..54b63b0 100644 --- a/docker-compose/Makefile +++ b/docker-compose/Makefile @@ -1,14 +1,26 @@ -# Need to start Toddserver twice because it crashs if RabbitMq is not ready +# Rabbitmq and ETC needs to be started first to make sure that todd server starts properly start: + docker-compose up -d rabbitmq etcd docker-compose up -d - docker-compose up -d toddserver stop: docker-compose down -agents: +init: + @echo "Create Groups agent-dc and agent-hq in Todd" + @docker exec -it dockercompose_toddserver_1 todd create /root/group-dc.yaml + @docker exec -it dockercompose_toddserver_1 todd create /root/group-hq.yaml + @echo "Create Tests in Todd" + @docker exec -it dockercompose_toddserver_1 todd create /root/test-ping-dns.yaml + @docker exec -it dockercompose_toddserver_1 todd create /root/test-ping.yaml + @docker exec -it dockercompose_toddserver_1 todd create /root/test-http.yaml + +show-agents: docker exec -it dockercompose_toddserver_1 todd agents +show-groups: + docker exec -it dockercompose_toddserver_1 todd groups + cli: docker exec -it dockercompose_toddserver_1 bash diff --git a/docker-compose/README.md b/docker-compose/README.md index e69de29..89569cb 100644 --- a/docker-compose/README.md +++ b/docker-compose/README.md @@ -0,0 +1,35 @@ + +# Start all components +By default it will start: The todd server, Rabbitmq, etcd and 2 todd agents +``` +make start +``` + +Both agents should register automatically, you can check using: +``` +✗ make agents +docker exec -it dockercompose_toddserver_1 todd agents +UUID EXPIRES ADDR FACT SUMMARY COLLECTOR SUMMARY +9c5992d8622e 23s 172.25.2.3 Addresses, Hostname get_addresses, get_hostname +7d9d0f0ec4b7 23s 172.25.1.3 Hostname, Addresses get_addresses, get_hostname +``` + +You can increase the number of agents with docker-compose using `docker-compose scale` +``` +docker-compose scale agent-hq=2 agent-dc=3 +``` +> New agents will register automatically, you can check using `make show-agents` + +# Create groups and tests in todd +Once all components are up, you need to create some `groups` and `tests`. +you can create the predefined groups and tests with: +``` +make init +``` +it will create 2 groups `agent-hq` and `agent-dc` and create 3 tests. + +# access the Todd Cli + +``` +make cli +``` diff --git a/docker-compose/objects/test-http.yaml b/docker-compose/objects/test-http.yaml index fc513fb..b19c174 100644 --- a/docker-compose/objects/test-http.yaml +++ b/docker-compose/objects/test-http.yaml @@ -5,7 +5,7 @@ label: test-http spec: targettype: uncontrolled source: - name: docker + name: agent-dc app: http args: "" target: diff --git a/docker-compose/objects/test-ping-dns.yaml b/docker-compose/objects/test-ping-dns.yaml index b7481af..7eb9b12 100644 --- a/docker-compose/objects/test-ping-dns.yaml +++ b/docker-compose/objects/test-ping-dns.yaml @@ -4,13 +4,10 @@ type: testrun label: test-ping-dns-dc spec: targettype: uncontrolled - target: + source: name: agent-dc app: ping args: "-c 10" - source: - name: agent-dc - target: - 4.2.2.2 - 8.8.8.8 From a9735f178592046bc9229c385a3633672d8e0cb9 Mon Sep 17 00:00:00 2001 From: Damien Garros Date: Tue, 21 Mar 2017 15:14:02 -0700 Subject: [PATCH 6/6] Add ping test between groups --- docker-compose/objects/test-ping.yaml | 14 ++++++++++++++ 1 file changed, 14 insertions(+) create mode 100644 docker-compose/objects/test-ping.yaml diff --git a/docker-compose/objects/test-ping.yaml b/docker-compose/objects/test-ping.yaml new file mode 100644 index 0000000..c273bea --- /dev/null +++ b/docker-compose/objects/test-ping.yaml @@ -0,0 +1,14 @@ +--- +# Example test file +type: testrun +label: test-ping +spec: + targettype: group + source: + name: agent-dc + app: ping + args: "-c 10" + target: + name: agent-hq + app: ping + args: "-c 10"