Commit aedd358d authored by Pavlos Daoglou's avatar Pavlos Daoglou
Browse files

Merge pull request #36 from pkoro/costumer_j2

ARGO-242 Costumer j2
parents 5007354b 2a7b4f6d
---
tenants:
TenantA:
topics:
- "probe.metricOutput.tenantA.ngi.*"
- "probe.metricOutput.tenantA.roc.*"
- "probe.metricOutput.tenantA.opsmonitor.*"
- "probe.metricOutput.tenantA.project.*"
- "probe.metricOutput.tenantA.vo.*"
brokers:
- "broker1.example.com"
- "broker2.example.com"
outputdir: "/var/lib/argo-connectors/TenantA/"
jobs_all: "JOB_TenantA_ALL, JOB_TenantA_PART"
jobs_details:
- name: "JOB_TenantA_ALL"
Directory: "TenantA_ALL"
Profiles: "ALL_SERVICES"
TopoType: "GOCDB"
TopoFeed: "https://goc.example.com/gocdbpi/"
TopoFetchType: "Sites"
TopoSelectGroupofEndpoints: "Production:Y, Monitored:Y, Scope:TenantA"
TopoSelectGroupOfGroups: "Certification:Certified, Infrastructure:Production, Scope:TenantA"
DowntimesFeed: "https://goc.example.com/gocdbpi/"
- name: "JOB_TenantA_PART"
Directory: "TenantA_PART"
Profiles: "PART_SERVICES"
TopoType: "GOCDB"
TopoFeed: "https://goc.example.com/gocdbpi/"
TopoFetchType: "Sites"
TopoSelectGroupofEndpoints: "Production:Y, Monitored:Y, Scope:TenantA"
TopoSelectGroupOfGroups: "Certification:Candidate, Infrastructure:Production, Scope:TenantA"
DowntimesFeed: "https://goc.example.com/gocdbpi/"
TenantB:
topics:
- "probe.*"
brokers:
- "broker3.example.com"
outputdir: "/var/lib/argo-connectors/TenantB/"
jobs_all: "JOB_TenantB_SERVICES"
jobs_details:
- name: "JOB_TenantB_SERVICES"
Directory: "SERVICES"
Profiles: "My_Critical_Services"
TopoType: "GOCDB"
TopoFeed: "https://goc.example.com/gocdbpi/"
TopoFetchType: "ServiceGroups"
TopoSelectGroupofEndpoints: "Production:Y, Monitored:Y, Scope:TenantB"
TopoSelectGroupOfGroups: "Certification:Candidate, Infrastructure:Production, Scope:TenantB"
DowntimesFeed: "https://goc.example.com/gocdbpi/"
poem_servers:
- host: "poemA.example.com"
vos:
- ops
- gridpp
- host: "poemB.example.com"
vos:
- ops
poem_fetch_profiles:
- profile_1
- profile_2
[DEFAULT]
GridPPFeed = http://www-pnp.physics.ox.ac.uk/%7Emohammad/gridpp.feed.xml
[CUSTOMER_EGI]
Name = EGI
OutputDir = /var/lib/argo-connectors/EGI/
Jobs = JOB_EGI-ROC, JOB_EGI-ROC-Critical, JOB_EGI-ROC-Operators, JOB_EGI-OPS-MONITOR, JOB_EGI-OPS-MONITOR-Critical, JOB_EGI-CLOUDMON, JOB_EGI-CLOUDMON-Critical, JOB_EGI-GLEXEC, JOB_GridPPTest
[JOB_EGI-ROC-Critical]
Dirname = EGI_Critical
Profiles = ROC_CRITICAL
TopoType = GOCDB
TopoFetchType = Sites
TopoSelectGroupOfEndpoints = Production:Y, Monitored:Y, Scope:EGI
TopoSelectGroupOfGroups = Certification:Certified, Infrastructure:Production, Scope:EGI
[JOB_EGI-ROC]
Dirname = EGI_ROC
Profiles = ROC
TopoType = GOCDB
TopoFetchType = Sites
TopoSelectGroupOfEndpoints = Production:Y, Monitored:Y, Scope:EGI
TopoSelectGroupOfGroups = Certification:Certified, Infrastructure:Production, Scope:EGI
[JOB_EGI-ROC-Operators]
Dirname = EGI_ROC-Operators
Profiles = ROC_OPERATORS
TopoType = GOCDB
TopoFetchType = Sites
TopoSelectGroupOfEndpoints = Production:Y, Monitored:Y, Scope:EGI
TopoSelectGroupOfGroups = Certification:Certified, Infrastructure:Production, Scope:EGI
[JOB_EGI-OPS-MONITOR-Critical]
Dirname = EGI_OPS-MONITOR-Critical
Profiles = OPS_MONITOR_CRITICAL
TopoType = GOCDB
TopoFetchType = Sites
TopoSelectGroupOfEndpoints = Production:Y, Monitored:Y, Scope:EGI
TopoSelectGroupOfGroups = Certification:Certified, Infrastructure:Production, Scope:EGI
[JOB_EGI-OPS-MONITOR]
Dirname = EGI_OPS-MONITOR
Profiles = OPS_MONITOR
TopoType = GOCDB
TopoFetchType = Sites
TopoSelectGroupOfEndpoints = Production:Y, Monitored:Y, Scope:EGI
TopoSelectGroupOfGroups = Certification:Certified, Infrastructure:Production, Scope:EGI
[JOB_EGI-CLOUDMON-Critical]
Dirname = EGI_CLOUDMON_Critical
Profiles = CLOUD-MON_CRITICAL
TopoType = GOCDB
TopoFetchType = Sites
TopoSelectGroupOfEndpoints = Monitored:Y, Scope:EGI
TopoSelectGroupOfGroups = Scope:EGI
[JOB_EGI-CLOUDMON]
Dirname = EGI_CLOUDMON
Profiles = CLOUD-MON
TopoType = GOCDB
TopoFetchType = Sites
TopoSelectGroupOfEndpoints = Monitored:Y, Scope:EGI
TopoSelectGroupOfGroups = Scope:EGI
[JOB_EGI-GLEXEC]
Dirname = EGI_GLEXEC
Profiles = GLEXEC
TopoType = GOCDB
TopoFetchType = Sites
TopoSelectGroupOfEndpoints = Production:Y, Monitored:Y, Scope:EGI
TopoSelectGroupOfGroups = Certification:Certified, Infrastructure:Production, Scope:EGI
[JOB_GridPPTest]
Dirname = GridPP_Testing
Profiles = TESTING
TopoType = VOFeed
TopoFeed = %(GridPPFeed)s
#TopoSelectGroupOfGroups = Type:OPS_Site
......@@ -3,3 +3,7 @@
- name: restart egi consumer
service: name=argo-egi-consumer state=restarted
- name: restart all consumers
service: name=argo-{{ item.key | lower }}-consumer state=restarted
with_dict: tenants
......@@ -12,32 +12,120 @@
tags: ar-packages
pip: name=pymongo state=latest
- name: Install consumer from ar project
tags: ar-packages
- name: Install egi consumer package from ar project
tags:
- ar-packages
- consumer_config
yum: name=argo-egi-consumer state=latest enablerepo={{ enabled_argo_repo }}
notify: restart egi consumer
notify: restart all consumers
- name: Create consumer configuration directories
file: path=/etc/argo-{{ item.key | lower }}-consumer
state=directory
owner=root group=root mode=0755
with_dict: tenants
notify: restart all consumers
- name: Copy metric avro specification for each tenant
tags:
- consumer_config
template: src=metric_data.avsc.j2
dest=/etc/argo-{{ item.key | lower }}-consumer/metric_data.avsc
owner=root group=root mode=0644
with_dict: tenants
notify: restart all consumers
- name: Consumer configuration
- name: Create consumer output directories per tenant
tags: consumer_config
file: path=/var/lib/argo-{{ item.key | lower }}-consumer
state=directory
owner=arstats group=arstats mode=0755
with_dict: tenants
- name: Consumer configuration
tags:
- consumer_config
template: src=consumer.conf.j2
dest=/etc/argo-egi-consumer/consumer.conf
dest=/etc/argo-{{ item.key | lower }}-consumer/consumer.conf
owner=root group=root mode=0644
- name: Enable and start consumer service
tags: consumer_config
service: name=argo-egi-consumer enabled=on state=started
with_dict: tenants
notify: restart all consumers
- name: Copy out init scripts for consumers
tags:
- consumer_config
template: src=consumer.init.j2
dest=/etc/init.d/argo-{{ item.key | lower }}-consumer
owner=root group=root mode=0755
with_dict: tenants
notify: restart all consumers
- name: Create copies of python wrappers for non egi consumers
tags:
- consumer_config
file: path=/usr/bin/argo-{{ item.key | lower }}-wrapper-consumer.py
state=link src=/usr/bin/argo-egi-consumer.py
with_dict: tenants
- name: Enable and start consumer services
tags:
- consumer_config
service: name=argo-{{ item.key | lower }}-consumer enabled=on state=started
with_dict: tenants
- name: Install argo-egi-connectors from ar project
tags: ar-packages
tags:
- ar-packages
- connectors
yum: name=argo-egi-connectors state=latest enablerepo={{ enabled_argo_repo }}
- name: Configure EGI connectors
tags: connectors_config
copy: src=customer.conf
dest=/etc/argo-egi-connectors/customer.conf
- name: Configure connectors
tags:
- connectors_config
- connectors
template: src=customer.conf.j2
dest=/etc/argo-egi-connectors/{{ item.key | lower }}-customer.conf
owner=root group=root mode=0644
backup=yes
with_dict: tenants
- name: POEM configuration
tags:
- connectors_config
- poem_config
template: src=poem-connector.conf.j2
dest=/etc/argo-egi-connectors/poem-connector.conf
owner=root group=root mode=0644
backup=yes
- name: Configure poem connector per tenant cron job
tags:
- connectors_config
- connectors
- poem_cron
cron: cron_file=poem_{{ item.key | lower }}
name=poem_{{ item.key | lower }}
minute=2
hour=0
user=root
job="/usr/libexec/argo-egi-connectors/poem-connector.py -c /etc/argo-egi-connectors/{{ item.key | lower }}-customer.conf"
state=present
with_dict: tenants
- name: Configure poem connector per tenant cron job
tags:
- connectors_config
- connectors
- topology_cron
cron: cron_file=topology_{{ item.key | lower }}
name=topology_{{ item.key | lower }}
minute=7
hour=0
user=root
job="/usr/libexec/argo-egi-connectors/topology-gocdb-connector.py -c /etc/argo-egi-connectors/{{ item.key | lower }}-customer.conf"
state=present
with_dict: tenants
- name: Install ar-compute from ar project
tags: ar-packages
yum: name=ar-compute state=latest enablerepo={{ enabled_argo_repo }}
......@@ -67,7 +155,7 @@
backup=yes
- name: Configure ar-compute stuff 4
tags: consumer_config
tags: compute_config
lineinfile: dest=/etc/ar-compute-engine.conf
regexp="^sync_path="
line="sync_path=/var/lib/argo-connectors"
......@@ -75,7 +163,7 @@
backup=yes
- name: Configure ar-compute stuff 5
tags: consumer_config
tags: compute_config
lineinfile: dest=/etc/ar-compute-engine.conf
regexp="^sync_exec="
line="sync_exec=/usr/libexec/argo-egi-connectors"
......@@ -112,20 +200,6 @@
job="/usr/libexec/ar-compute/bin/poller_ar.py -t EGI"
state=present
- name: Create job directories
tags: sync_config
file: path={{ item }} owner=root group=root mode=0755 state=directory
with_items:
- /var/lib/argo-connectors/EGI/Cloudmon
- /var/lib/argo-connectors/EGI/Critical
- name: Make sure ownerships are OK
tags: consumer_config
file: path={{ item }} owner=root group=arstats mode=0775 state=directory
with_items:
- /var/lib/argo-connectors
- /var/lib/argo-egi-consumer
- name: Install ar-data-retention from ar project
tags: ar-data-retention
yum: name=ar-data-retention state=latest enablerepo={{ enabled_argo_repo }}
......
[General]
LogFaultyTimestamps = True
ReportWritMsgEveryHours = 24
AvroSchema = /etc/argo-egi-consumer/metric_data.avsc
AvroSchema = /etc/argo-{{ item.key | lower }}-consumer/metric_data.avsc
[Subscription]
Destinations = /queue/Consumer.{{ ansible_fqdn|replace(".","_") }}.grid.probe.metricOutput.EGEE.ngi.*, /queue/Consumer.{{ ansible_fqdn|replace(".","_") }}.grid.probe.metricOutput.EGEE.roc.*, /queue/Consumer.{{ ansible_fqdn|replace(".","_") }}.grid.probe.metricOutput.EGEE.opsmonitor.*, /queue/Consumer.{{ ansible_fqdn|replace(".","_") }}.grid.probe.metricOutput.EGEE.project.*, /queue/Consumer.{{ ansible_fqdn|replace(".","_") }}.grid.probe.metricOutput.EGEE.vo.*
Destinations = {% if item.value.topics is defined %}{% for topic in item.value.topics %}{% if loop.last %}/queue/Consumer.{{ ansible_fqdn|replace(".","_") }}.{{ topic }}{% else %}/queue/Consumer.{{ ansible_fqdn|replace(".","_") }}.{{ topic }}, {% endif %}{% endfor %}{% endif %}
IdleMsgTimeout = 0
[Brokers]
Server1 = mq.cro-ngi.hr:6163
Server2 = mq.afroditi.hellasgrid.gr:6163
{% if item.value.brokers is defined %}
{% for broker in item.value.brokers %}
Server{{ loop.index }} = {{ broker }}:6163
{% endfor %}
{% endif %}
[Authentication]
HostKey = /etc/grid-security/hostkey.pem
......@@ -26,6 +30,6 @@ UseSSL = False
[Output]
WritePlaintext = False
Directory = /var/lib/argo-egi-consumer
Directory = /var/lib/argo-{{ item.key | lower }}-consumer
Filename = argo-consumer_log_%s.avro
ErrorFilename = argo-consumer_error_log_%s.avro
\ No newline at end of file
#!/bin/bash
# argo-{{ item.key | lower }}-consumer daemon
# chkconfig: 345 20 80
# description: argo-{{ item.key | lower }}-consumer daemon
# processname: argo-{{ item.key | lower }}-consumer
. /etc/rc.d/init.d/functions
PROG_NAME="argo-{{ item.key | lower }}-consumer"
CONFIG="/etc/argo-{{ item.key | lower }}-consumer/consumer.conf"
DAEMON_PATH="/usr/bin/argo-{{ item.key | lower }}-wrapper-consumer.py"
case "$1" in
start)
echo -n $"Starting $PROG_NAME: "
daemon $DAEMON_PATH --start --config $CONFIG
echo
;;
status)
$DAEMON_PATH --status --config $CONFIG
;;
stop)
echo -n $"Stopping $PROG_NAME: "
daemon $DAEMON_PATH --stop --config $CONFIG
echo
;;
restart)
echo -n $"Restarting $PROG_NAME: "
daemon $DAEMON_PATH --restart --config $CONFIG
echo
;;
*)
echo "Usage: $0 {status|start|stop|restart}"
exit 1
esac
[CUSTOMER_{{item.key|upper}}]
Name = {{item.key|upper}}
OutputDir = {{item.value.outputdir}}
{% if item.value.jobs_all is defined %}
Jobs = {{ item.value.jobs_all }}
{% endif %}
{% if item.value.jobs_details is defined %}
{% for job in item.value.jobs_details %}
[{{job.name}}]
Dirname = {{job.Directory}}
Profiles = {{job.Profiles}}
TopoType = {{job.TopoType}}
TopoFeed = {{job.TopoFeed}}
TopoFetchType = {{job.TopoFetchType}}
TopoSelectGroupofEndpoints = {{job.TopoSelectGroupofEndpoints}}
TopoSelectGroupOfGroups = {{job.TopoSelectGroupOfGroups}}
DowntimesFeed = {{job.DowntimesFeed}}
{% endfor %}
{% endif %}
{"namespace": "argo.avro",
"type": "record",
"name": "metric_data",
"fields": [
{"name": "timestamp", "type": "string"},
{"name": "service", "type": "string"},
{"name": "hostname", "type": "string"},
{"name": "metric", "type": "string"},
{"name": "status", "type": "string"},
{"name": "monitoring_host", "type": ["null", "string"]},
{"name": "summary", "type": ["null", "string"]},
{"name": "message", "type": ["null", "string"]},
{"name": "tags", "type" : ["null", {"name" : "Tags",
"type" : "map",
"values" : ["null", "string"]
}]
}]
}
{% for poem in poem_servers %}
[PoemServer{{ loop.index }}]
Host = {{ poem.host }}
VO = {{ poem.vos|join(', ') }}
{% endfor %}
[FetchProfiles]
List = {{ poem_fetch_profiles|join(', ')|upper }}
[PrefilterData]
AllowedNGI = http://mon.egi.eu/nagios-roles.conf
AllowedNGIProfiles = ch.cern.sam.ROC, ch.cern.sam.ROC_OPERATORS, ch.cern.sam.ROC_CRITICAL, ch.cern.sam.GLEXEC, eu.eudat.argo.CRITICAL
AllNGI1 = opsmon.egi.eu
AllNGIProfiles1 = ch.cern.sam.OPS_MONITOR, ch.cern.sam.OPS_MONITOR_CRITICAL
AllNGI2 = cloudmon.egi.eu
AllNGIProfiles2 = ch.cern.sam.CLOUD-MON, ch.cern.sam.CLOUD-MON_CRITICAL
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment