Commit e9dfeecc authored by Christos Kanellopoulos's avatar Christos Kanellopoulos
Browse files

Merge pull request #18 from ARGOeu/devel

Sprint 29
parents 1f0b34ab 33ad4abc
## ARGO Ansible
# ARGO via Ansible
Contains Ansible playbook for the deployment of the ARGO datastore and API services. The play is split into four roles:
- repos (includes tasks for the installation of the required repository definitions
## WebAPI deployment
Contains Ansible playbook for the deployment of the ARGO datastore and API service. The play is split into four (4) roles:
- repos (includes tasks for the installation of the required repository definitions)
- has_certificate (task for uploading the certificate file onto the host under the appropriate path)
- mongodb (installation and configuration of mongodb datastore)
- webapi (installation and bootstrap of ARGO api service)
### Things to do before deployment
- Obtain a key/certificate pair from a trusted CA and after place them both under roles/has_certificate/files.
- Obtain a key/certificate pair from a trusted CA and after place them both under roles/has_certificate/files with names `hostkey.pem` and `hostcert.pem` respectively.
- Edit inventory and replace `webapi.node` with the hostname that you intend to deploy the API onto.
### Prerequisites
- Deploy against CentOS 6.x node
- Make sure `libselinux-python` is installed on the target node
- Ansible version used is `1.7.2`
### How to deploy
```bash
$ ansible-playbook -v webapi.yml
```
\ No newline at end of file
```
## Full standalone deployment
Contains Ansible playbook for the deployment of all ARGO components. The play is split into six (6) roles:
- repos (includes tasks for the installation of the required repository definitions)
- ca_bundle (includes a task for the installation of the egi-ca-policy-core bundle)
- has_certificate (task for uploading the certificate file onto the host under the appropriate path)
- consumer (includes tasks for the installation of the ARGO consumer and feed components)
- mongodb (installation and configuration of mongodb datastore)
- webapi (installation and bootstrap of ARGO api service)
### Things to do before deployment
- Obtain a key/certificate pair from a trusted CA and after place them both under roles/has_certificate/files with names `hostkey.pem` and `hostcert.pem` respectively.
- Edit inventory and replace `standalone.node` with the hostname that you intend to deploy the complete ARGO stack onto.
### Prerequisites
- Deploy against CentOS 6.x node
- Make sure `libselinux-python` is installed on the target node
- Ansible version used is `1.7.2`
### How to deploy
```bash
$ ansible-playbook -v standalone.yml
```
......@@ -4,4 +4,4 @@ epel_release_url: http://ftp.ntua.gr/pub/linux/fedora-epel/6/i386/
epel_release_name: epel-release-6-8.noarch.rpm
arstats_release_url: http://rpm.hellasgrid.gr/mash/centos6-arstats/i386/
arstats_release_name: ar-release-1.0.0-3.25.el6.noarch.rpm
arstats_release_name: ar-release-1.0.0-3.el6.noarch.rpm
---
mongo_bind_interface: 127.0.0.1
cert_path: /etc/grid-security/hostcert.pem
key_path: /etc/grid-security/hostkey.pem
iptables_rules:
input:
- { dport: "443", proto: "tcp", policy: "accept"}
hive_retention_in_days: 900
mongo_retention_in_days: 900
files_retention_in_days: 900
---
mongo_bind_interface: 0.0.0.0
cert_path: /etc/pki/tls/certs/localhost.crt
key_path: /etc/pki/tls/private/localhost.key
iptables_rules:
input:
- { dport: "443", proto: "tcp", policy: "accept"}
- { dport: "27017", proto: "tcp", policy: "accept"}
\ No newline at end of file
......@@ -2,3 +2,5 @@
[webapi]
webapi.node
[standalone]
standalone.node
---
- name: Install EGI CA certificates
tags: ca_bundle
yum: name=ca-policy-egi-core state=latest
---
- name: restart consumer
service: name=ar-consumer state=restarted
---
- name: Install consumer from ar project
tags: ar-packages
yum: name=ar-consumer state=latest
notify: restart consumer
- name: Install packages from ar project
tags: ar-packages
yum: name={{ item }} state=latest
with_items:
- ar-sync
- ar-compute
- name: Configure ar-compute stuff 1
tags: compute_config
lineinfile: dest=/etc/ar-compute-engine.conf
regexp="^mongo_host="
line="mongo_host=127.0.0.1"
state=present
backup=yes
- name: Configure ar-compute stuff 2
tags: compute_config
lineinfile: dest=/etc/ar-compute-engine.conf
regexp="^mode="
line="mode=local"
state=present
backup=yes
- name: Configure ar-compute stuff 3
tags: compute_config
lineinfile: dest=/etc/ar-compute-engine.conf
regexp="^prefilter_clean="
line="prefilter_clean=false"
state=present
backup=yes
- name: Configure ar-compute job cycle daily cron
tags: compute_config
cron: cron_file=ar_job_cycle_daily
user=root
name=job_cycle_daily
state=present
minute=0
hour=0
job="/usr/libexec/ar-compute/bin/job_cycle.py -d $(/bin/date --utc --date '-1 day' +\%Y-\%m-\%d)"
- name: Configure ar-compute job cycle hourly cron
tags: compute_config
cron: cron_file=ar_job_cycle_hourly
user=root
name=ar_job_cycle_hourly
state=present
minute=55
hour=*
job="/usr/libexec/ar-compute/bin/job_cycle.py -d $(/bin/date --utc +\%Y-\%m-\%d)"
- name: Create job directories
tags: sync_config
file: path={{ item }} owner=root group=root mode=0755 state=directory
with_items:
- /var/lib/ar-sync/EGI/Cloudmon
- /var/lib/ar-sync/EGI/Critical
- name: Make sure ownerships are OK
tags: consumer_config
file: path={{ item }} owner=root group=arstats mode=0775 state=directory
with_items:
- /var/lib/ar-sync
- /var/lib/ar-consumer
- name: Enable and start consumer service
tags: consumer_config
service: name=ar-consumer enabled=yes state=started
- name: Install ar-data-retention from ar project
tags: ar-data-retention
yum: name=ar-data-retention state=latest
- name: Parametrize data retention policies
tags: data_retention
template: src=ar-data-retention.conf.j2
dest=/etc/ar-data-retention/ar-data-retention.conf
owner=root group=root mode=0644
#######################################
# HIVE config
######################################
[hive]
# hive tables to delete data from
hiveTables: raw_data
# hive date field name for each table
hiveDateFields: dates
# hive date field format
hiveDateFormat: %Y%m%d
#hive retention
hiveRetentionInDays: {{ hive_retention_in_days }}
########################################
# MongoDB config
######################################
[mongo]
# MongoDB server
mongoDBServer: mongoDBhost
# MongoDB service port
mongoDBServerPort: 27017
# MongoDB database
mongoDBDatabase: AR
# MongoDB tables to delete data from
mongoDBTables: sites;sfreports;status_endpoints;status_metric;status_services;status_sites
# MongoDB date field name for each table
mongoDBDateFields: dt;dt;di;di;di;di
# MongoDB date field format for each table
mongoDBDateFieldTypes: int;int;int;int;int;int
# MongoDB date field format
mongoDBDateFormat: %Y%m%d
# MongoDB retention
mongoDBRetentionInDays: {{ mongo_retention_in_days }}
#######################################
# Files config
#######################################
[file]
fileTemplate_1: /var/lib/ar-consumer/ar-consumer_log_%Y_%m_%d.txt
fileTemplate_2: /var/lib/ar-consumer/ar-consumer_log_%Y_%m_%d.avro
fileTemplate_3: /var/lib/ar-consumer/ar-consumer_error_%Y_%m_%d.txt
fileTemplate_4: /var/lib/ar-sync/downtimes_%Y-%m-%d.out
fileTemplate_5: /var/lib/ar-sync/downtimes_%Y-%m-%d.avro
fileTemplate_6: /var/lib/ar-sync/poem_sync_%Y_%m_%d.out
fileTemplate_7: /var/lib/ar-sync/poem_sync_%Y_%m_%d.avro
fileTemplate_8: /var/lib/ar-sync/prefilter_%Y_%m_%d.out
fileTemplate_9: /var/lib/ar-sync/prefilter_%Y_%m_%d.avro
fileTemplate_10: /var/lib/ar-sync/hepspec_sync_%Y_%m_%d.out
fileTemplate_11: /var/lib/ar-sync/weights_sync_%Y_%m_%d.avro
fileTemplate_12: /var/lib/ar-sync/sites_%Y_%m_%d.out
fileTemplate_13: /var/lib/ar-sync/sites_%Y_%m_%d.avro
fileTemplate_14: /var/lib/ar-sync/groups_%Y_%m_%d.out
fileTemplate_15: /var/lib/ar-sync/groups_%Y_%m_%d.avro
fileRetentionInDays: {{ files_retention_in_days }}
---
- name: reload iptables
service: name=iptables state=reloaded
- name: reload ip6tables
service: name=ip6tables state=reloaded
---
- name: copy iptables file onto host
template: src=iptables.j2
dest=/etc/sysconfig/iptables backup=no
owner=root group=root mode=0600
when: iptables_rules is defined
notify: reload iptables
- name: copy ip6tables file onto host
template: src=ip6tables.j2
dest=/etc/sysconfig/ip6tables backup=no
owner=root group=root mode=0600
when: ip6tables_rules is defined
notify: reload ip6tables
# Firewall configuration written by Ansible
# Manual customization of this file is not recommended.
*filter
:INPUT ACCEPT [0:0]
:FORWARD ACCEPT [0:0]
:OUTPUT ACCEPT [0:0]
-A INPUT -m state --state ESTABLISHED,RELATED -j ACCEPT
-A INPUT -p ipv6-icmp -j ACCEPT
-A INPUT -i lo -j ACCEPT
-A INPUT -m state --state NEW -m tcp -p tcp --dport 22 -j ACCEPT
{% if ip6tables_rules.input is defined %}
{% for rule in ip6tables_rules.input %}
-A INPUT -m state --state NEW {% if rule.source is defined %}-s {{ rule.source }}{% endif %} {% if rule.proto is defined %}-m {{ rule.proto }} -p {{ rule.proto }}{% endif %} {% if rule.dport is defined %}--dport {{ rule.dport }}{% endif %} -j {{ rule.policy | upper }}
{% endfor %}
{% endif %}
-A INPUT -j REJECT --reject-with icmp6-adm-prohibited
-A FORWARD -j REJECT --reject-with icmp6-adm-prohibited
-A OUTPUT -m state --state ESTABLISHED,RELATED -j ACCEPT
{% if ip6tables_rules.output is defined %}
{% for rule in ip6tables_rules.output %}
-A OUTPUT -m state --state NEW {% if rule.dest is defined %}-d {{ rule.dest }}{% endif %} {% if rule.proto is defined %}-m {{ rule.proto }} -p {{ rule.proto }}{% endif %} {% if rule.dport is defined %}--dport {{ rule.dport }}{% endif %} -j {{ rule.policy | upper }}
{% endfor %}
{% endif %}
COMMIT
# Firewall configuration written by Ansible
# Manual customization of this file is not recommended.
*filter
:INPUT ACCEPT [0:0]
:FORWARD ACCEPT [0:0]
:OUTPUT ACCEPT [0:0]
-A INPUT -m state --state ESTABLISHED,RELATED -j ACCEPT
-A INPUT -p icmp -j ACCEPT
-A INPUT -i lo -j ACCEPT
-A INPUT -m state --state NEW -m tcp -p tcp --dport 22 -j ACCEPT
{% if iptables_rules.input is defined %}
{% for rule in iptables_rules.input %}
-A INPUT -m state --state NEW {% if rule.source is defined %}-s {{ rule.source }}{% endif %} {% if rule.proto is defined %}-m {{ rule.proto }} -p {{ rule.proto }}{% endif %} {% if rule.dport is defined %}--dport {{ rule.dport }}{% endif %} -j {{ rule.policy | upper }}
{% endfor %}
{% endif %}
-A INPUT -j REJECT --reject-with icmp-host-prohibited
-A FORWARD -j REJECT --reject-with icmp-host-prohibited
-A OUTPUT -m state --state ESTABLISHED,RELATED -j ACCEPT
-A OUTPUT -p icmp -j ACCEPT
{% if iptables_rules.output is defined %}
{% for rule in iptables_rules.output %}
-A OUTPUT -m state --state NEW {% if rule.dest is defined %}-d {{ rule.dest }}{% endif %} {% if rule.proto is defined %}-m {{ rule.proto }} -p {{ rule.proto }}{% endif %} {% if rule.dport is defined %}--dport {{ rule.dport }}{% endif %} -j {{ rule.policy | upper }}
{% endfor %}
{% endif %}
COMMIT
......@@ -5,3 +5,22 @@
copy: src=hostcert.pem
dest={{ cert_path }} backup=yes
owner=root group=root mode=0644
- name: Copy host x509 key onto host
tags: certificate
copy: src=hostkey.pem
dest={{ key_path }} backup=yes
owner=root group=root mode=0400
- name: Create softlink (to cert) for API
file: state=link
src={{ cert_path }}
path=/etc/pki/tls/certs/localhost.crt
when: inventory_hostname in groups.standalone
- name: Create softlink (to key) for API
file: state=link
src={{ key_path }}
path=/etc/pki/tls/private/localhost.key
when: inventory_hostname in groups.standalone
mongod soft nproc unlimited
\ No newline at end of file
......@@ -11,10 +11,15 @@
lineinfile: dest=/etc/mongod.conf
regexp="^bind_ip="
insertafter=EOF
line="bind_ip=0.0.0.0"
line="bind_ip={{ mongo_bind_interface }}"
state=present
backup=yes
notify: restart mongo
- name: Increase soft nproc limits
copy: src=etc/security/limits.d/99-mongodb-nproc.conf
dest=/etc/security/limits.d/99-mongodb-nproc.conf backup=no
owner=root group=root mode=0644
- name: Enable and start mongoDB
service: name=mongod enabled=yes state=started
\ No newline at end of file
- name: Configure pig related parameter
tags: pig_client_config
lineinfile: dest=/etc/pig/conf/pig.properties
regexp="^pig.logfile"
line="pig.logfile=/tmp/pig-err.log"
state=present
backup=yes
- name: Insert comment for pig related parameter
tags: pig_client_config
lineinfile: dest=/etc/pig/conf/pig.properties
regexp="^# File Parameter"
insertbefore="^pig.logfile"
line="# File Parameter for pig exception dump."
state=present
backup=yes
[EGI-trustanchors]
name=EGI-trustanchors
baseurl=http://repository.egi.eu/sw/production/cas/1/current/
enabled=1
gpgcheck=1
gpgkey=http://repository.egi.eu/sw/production/cas/1/GPG-KEY-EUGridPMA-RPM-3
[cloudera-cdh4]
name=Cloudera's Distribution for Hadoop, Version 4
baseurl=http://archive.cloudera.com/cdh4/redhat/$releasever/$basearch/cdh/4/
gpgkey = http://archive.cloudera.com/cdh4/redhat/$releasever/$basearch/cdh/RPM-GPG-KEY-cloudera
gpgcheck = 1
enabled = 1
[mongodb]
name=MongoDB Repository
baseurl=http://downloads-distro.mongodb.org/repo/redhat/os/x86_64/
gpgcheck=0
enabled=1
\ No newline at end of file
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment