Commit 2a7b4f6d authored by Paschalis Korosoglou's avatar Paschalis Korosoglou
Browse files

ARGO-242 Make connectors and consumer tenant aware.

parent d780eedd
......@@ -2,10 +2,19 @@
tenants:
TenantA:
outputdir: "/var/lib/argo-connectors/tenant_a/"
topics:
- "probe.metricOutput.tenantA.ngi.*"
- "probe.metricOutput.tenantA.roc.*"
- "probe.metricOutput.tenantA.opsmonitor.*"
- "probe.metricOutput.tenantA.project.*"
- "probe.metricOutput.tenantA.vo.*"
brokers:
- "broker1.example.com"
- "broker2.example.com"
outputdir: "/var/lib/argo-connectors/TenantA/"
jobs_all: "JOB_TenantA_ALL, JOB_TenantA_PART"
jobs_details:
- name: "ALL"
- name: "JOB_TenantA_ALL"
Directory: "TenantA_ALL"
Profiles: "ALL_SERVICES"
TopoType: "GOCDB"
......@@ -14,7 +23,7 @@ tenants:
TopoSelectGroupofEndpoints: "Production:Y, Monitored:Y, Scope:TenantA"
TopoSelectGroupOfGroups: "Certification:Certified, Infrastructure:Production, Scope:TenantA"
DowntimesFeed: "https://goc.example.com/gocdbpi/"
- name: "PART"
- name: "JOB_TenantA_PART"
Directory: "TenantA_PART"
Profiles: "PART_SERVICES"
TopoType: "GOCDB"
......@@ -24,10 +33,14 @@ tenants:
TopoSelectGroupOfGroups: "Certification:Candidate, Infrastructure:Production, Scope:TenantA"
DowntimesFeed: "https://goc.example.com/gocdbpi/"
TenantB:
outputdir: "/var/lib/argo-connectors/tenant_b/"
topics:
- "probe.*"
brokers:
- "broker3.example.com"
outputdir: "/var/lib/argo-connectors/TenantB/"
jobs_all: "JOB_TenantB_SERVICES"
jobs_details:
- name: "SERVICES"
- name: "JOB_TenantB_SERVICES"
Directory: "SERVICES"
Profiles: "My_Critical_Services"
TopoType: "GOCDB"
......@@ -36,3 +49,17 @@ tenants:
TopoSelectGroupofEndpoints: "Production:Y, Monitored:Y, Scope:TenantB"
TopoSelectGroupOfGroups: "Certification:Candidate, Infrastructure:Production, Scope:TenantB"
DowntimesFeed: "https://goc.example.com/gocdbpi/"
poem_servers:
- host: "poemA.example.com"
vos:
- ops
- gridpp
- host: "poemB.example.com"
vos:
- ops
poem_fetch_profiles:
- profile_1
- profile_2
......@@ -3,3 +3,7 @@
- name: restart egi consumer
service: name=argo-egi-consumer state=restarted
- name: restart all consumers
service: name=argo-{{ item.key | lower }}-consumer state=restarted
with_dict: tenants
......@@ -12,33 +12,120 @@
tags: ar-packages
pip: name=pymongo state=latest
- name: Install consumer from ar project
tags: ar-packages
- name: Install egi consumer package from ar project
tags:
- ar-packages
- consumer_config
yum: name=argo-egi-consumer state=latest enablerepo={{ enabled_argo_repo }}
notify: restart egi consumer
notify: restart all consumers
- name: Consumer configuration
- name: Create consumer configuration directories
file: path=/etc/argo-{{ item.key | lower }}-consumer
state=directory
owner=root group=root mode=0755
with_dict: tenants
notify: restart all consumers
- name: Copy metric avro specification for each tenant
tags:
- consumer_config
template: src=metric_data.avsc.j2
dest=/etc/argo-{{ item.key | lower }}-consumer/metric_data.avsc
owner=root group=root mode=0644
with_dict: tenants
notify: restart all consumers
- name: Create consumer output directories per tenant
tags: consumer_config
file: path=/var/lib/argo-{{ item.key | lower }}-consumer
state=directory
owner=arstats group=arstats mode=0755
with_dict: tenants
- name: Consumer configuration
tags:
- consumer_config
template: src=consumer.conf.j2
dest=/etc/argo-egi-consumer/consumer.conf
dest=/etc/argo-{{ item.key | lower }}-consumer/consumer.conf
owner=root group=root mode=0644
with_dict: tenants
notify: restart all consumers
- name: Copy out init scripts for consumers
tags:
- consumer_config
template: src=consumer.init.j2
dest=/etc/init.d/argo-{{ item.key | lower }}-consumer
owner=root group=root mode=0755
with_dict: tenants
notify: restart all consumers
- name: Enable and start consumer service
tags: consumer_config
service: name=argo-egi-consumer enabled=on state=started
- name: Create copies of python wrappers for non egi consumers
tags:
- consumer_config
file: path=/usr/bin/argo-{{ item.key | lower }}-wrapper-consumer.py
state=link src=/usr/bin/argo-egi-consumer.py
with_dict: tenants
- name: Enable and start consumer services
tags:
- consumer_config
service: name=argo-{{ item.key | lower }}-consumer enabled=on state=started
with_dict: tenants
- name: Install argo-egi-connectors from ar project
tags: ar-packages
tags:
- ar-packages
- connectors
yum: name=argo-egi-connectors state=latest enablerepo={{ enabled_argo_repo }}
- name: Configure connectors
tags: connectors_config
tags:
- connectors_config
- connectors
template: src=customer.conf.j2
dest=/etc/argo-egi-connectors/{{ item.key }}-customer.conf
dest=/etc/argo-egi-connectors/{{ item.key | lower }}-customer.conf
owner=root group=root mode=0644
backup=yes
with_dict: tenants
- name: POEM configuration
tags:
- connectors_config
- poem_config
template: src=poem-connector.conf.j2
dest=/etc/argo-egi-connectors/poem-connector.conf
owner=root group=root mode=0644
backup=yes
- name: Configure poem connector per tenant cron job
tags:
- connectors_config
- connectors
- poem_cron
cron: cron_file=poem_{{ item.key | lower }}
name=poem_{{ item.key | lower }}
minute=2
hour=0
user=root
job="/usr/libexec/argo-egi-connectors/poem-connector.py -c /etc/argo-egi-connectors/{{ item.key | lower }}-customer.conf"
state=present
with_dict: tenants
- name: Configure poem connector per tenant cron job
tags:
- connectors_config
- connectors
- topology_cron
cron: cron_file=topology_{{ item.key | lower }}
name=topology_{{ item.key | lower }}
minute=7
hour=0
user=root
job="/usr/libexec/argo-egi-connectors/topology-gocdb-connector.py -c /etc/argo-egi-connectors/{{ item.key | lower }}-customer.conf"
state=present
with_dict: tenants
- name: Install ar-compute from ar project
tags: ar-packages
yum: name=ar-compute state=latest enablerepo={{ enabled_argo_repo }}
......@@ -68,7 +155,7 @@
backup=yes
- name: Configure ar-compute stuff 4
tags: consumer_config
tags: compute_config
lineinfile: dest=/etc/ar-compute-engine.conf
regexp="^sync_path="
line="sync_path=/var/lib/argo-connectors"
......@@ -76,7 +163,7 @@
backup=yes
- name: Configure ar-compute stuff 5
tags: consumer_config
tags: compute_config
lineinfile: dest=/etc/ar-compute-engine.conf
regexp="^sync_exec="
line="sync_exec=/usr/libexec/argo-egi-connectors"
......@@ -113,20 +200,6 @@
job="/usr/libexec/ar-compute/bin/poller_ar.py -t EGI"
state=present
- name: Create job directories
tags: sync_config
file: path={{ item }} owner=root group=root mode=0755 state=directory
with_items:
- /var/lib/argo-connectors/EGI/Cloudmon
- /var/lib/argo-connectors/EGI/Critical
- name: Make sure ownerships are OK
tags: consumer_config
file: path={{ item }} owner=root group=arstats mode=0775 state=directory
with_items:
- /var/lib/argo-connectors
- /var/lib/argo-egi-consumer
- name: Install ar-data-retention from ar project
tags: ar-data-retention
yum: name=ar-data-retention state=latest enablerepo={{ enabled_argo_repo }}
......
[General]
LogFaultyTimestamps = True
ReportWritMsgEveryHours = 24
AvroSchema = /etc/argo-egi-consumer/metric_data.avsc
AvroSchema = /etc/argo-{{ item.key | lower }}-consumer/metric_data.avsc
[Subscription]
Destinations = /queue/Consumer.{{ ansible_fqdn|replace(".","_") }}.grid.probe.metricOutput.EGEE.ngi.*, /queue/Consumer.{{ ansible_fqdn|replace(".","_") }}.grid.probe.metricOutput.EGEE.roc.*, /queue/Consumer.{{ ansible_fqdn|replace(".","_") }}.grid.probe.metricOutput.EGEE.opsmonitor.*, /queue/Consumer.{{ ansible_fqdn|replace(".","_") }}.grid.probe.metricOutput.EGEE.project.*, /queue/Consumer.{{ ansible_fqdn|replace(".","_") }}.grid.probe.metricOutput.EGEE.vo.*
Destinations = {% if item.value.topics is defined %}{% for topic in item.value.topics %}{% if loop.last %}/queue/Consumer.{{ ansible_fqdn|replace(".","_") }}.{{ topic }}{% else %}/queue/Consumer.{{ ansible_fqdn|replace(".","_") }}.{{ topic }}, {% endif %}{% endfor %}{% endif %}
IdleMsgTimeout = 0
[Brokers]
Server1 = mq.cro-ngi.hr:6163
Server2 = mq.afroditi.hellasgrid.gr:6163
{% if item.value.brokers is defined %}
{% for broker in item.value.brokers %}
Server{{ loop.index }} = {{ broker }}:6163
{% endfor %}
{% endif %}
[Authentication]
HostKey = /etc/grid-security/hostkey.pem
......@@ -26,6 +30,6 @@ UseSSL = False
[Output]
WritePlaintext = False
Directory = /var/lib/argo-egi-consumer
Directory = /var/lib/argo-{{ item.key | lower }}-consumer
Filename = argo-consumer_log_%s.avro
ErrorFilename = argo-consumer_error_log_%s.avro
\ No newline at end of file
#!/bin/bash
# argo-{{ item.key | lower }}-consumer daemon
# chkconfig: 345 20 80
# description: argo-{{ item.key | lower }}-consumer daemon
# processname: argo-{{ item.key | lower }}-consumer
. /etc/rc.d/init.d/functions
PROG_NAME="argo-{{ item.key | lower }}-consumer"
CONFIG="/etc/argo-{{ item.key | lower }}-consumer/consumer.conf"
DAEMON_PATH="/usr/bin/argo-{{ item.key | lower }}-wrapper-consumer.py"
case "$1" in
start)
echo -n $"Starting $PROG_NAME: "
daemon $DAEMON_PATH --start --config $CONFIG
echo
;;
status)
$DAEMON_PATH --status --config $CONFIG
;;
stop)
echo -n $"Stopping $PROG_NAME: "
daemon $DAEMON_PATH --stop --config $CONFIG
echo
;;
restart)
echo -n $"Restarting $PROG_NAME: "
daemon $DAEMON_PATH --restart --config $CONFIG
echo
;;
*)
echo "Usage: $0 {status|start|stop|restart}"
exit 1
esac
{"namespace": "argo.avro",
"type": "record",
"name": "metric_data",
"fields": [
{"name": "timestamp", "type": "string"},
{"name": "service", "type": "string"},
{"name": "hostname", "type": "string"},
{"name": "metric", "type": "string"},
{"name": "status", "type": "string"},
{"name": "monitoring_host", "type": ["null", "string"]},
{"name": "summary", "type": ["null", "string"]},
{"name": "message", "type": ["null", "string"]},
{"name": "tags", "type" : ["null", {"name" : "Tags",
"type" : "map",
"values" : ["null", "string"]
}]
}]
}
{% for poem in poem_servers %}
[PoemServer{{ loop.index }}]
Host = {{ poem.host }}
VO = {{ poem.vos|join(', ') }}
{% endfor %}
[FetchProfiles]
List = {{ poem_fetch_profiles|join(', ')|upper }}
[PrefilterData]
AllowedNGI = http://mon.egi.eu/nagios-roles.conf
AllowedNGIProfiles = ch.cern.sam.ROC, ch.cern.sam.ROC_OPERATORS, ch.cern.sam.ROC_CRITICAL, ch.cern.sam.GLEXEC, eu.eudat.argo.CRITICAL
AllNGI1 = opsmon.egi.eu
AllNGIProfiles1 = ch.cern.sam.OPS_MONITOR, ch.cern.sam.OPS_MONITOR_CRITICAL
AllNGI2 = cloudmon.egi.eu
AllNGIProfiles2 = ch.cern.sam.CLOUD-MON, ch.cern.sam.CLOUD-MON_CRITICAL
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment