Commit 3bf3d07d authored by root's avatar root
Browse files

Migrating rs-stats from phab to gitlab

parent cd6584cc
import smtplib
from email.mime.text import MIMEText
smtp_port = 25
sender = 'nic@gr-ix.gr'
receivers = ['mdimolianis@netmode.ntua.gr','dimolianis.marinos@gmail.com']
def send_email(sender,receivers,subject,text):
msg = MIMEText(str(text))
msg['Subject'] = subject
msg['From'] = sender
msg['To'] = ','.join(receivers)
smtp_server = smtplib.SMTP('localhost', smtp_port)
smtp_server.sendmail(sender, receivers, msg.as_string())
#send_email(sender,receivers,"Test","Test")
RSasn = 50745
IXP_LC_FILTERED_PREFIX_LEN_TOO_LONG = ( RSasn, 1101, 1 )
IXP_LC_FILTERED_PREFIX_LEN_TOO_SHORT = ( RSasn, 1101, 2 )
IXP_LC_FILTERED_BOGON = ( RSasn, 1101, 3 )
IXP_LC_FILTERED_BOGON_ASN = ( RSasn, 1101, 4 )
IXP_LC_FILTERED_AS_PATH_TOO_LONG = ( RSasn, 1101, 5 )
IXP_LC_FILTERED_AS_PATH_TOO_SHORT = ( RSasn, 1101, 6 )
IXP_LC_FILTERED_FIRST_AS_NOT_PEER_AS = ( RSasn, 1101, 7 )
IXP_LC_FILTERED_NEXT_HOP_NOT_PEER_IP = ( RSasn, 1101, 8 )
IXP_LC_FILTERED_IRRDB_PREFIX_FILTERED = ( RSasn, 1101, 9 )
IXP_LC_FILTERED_IRRDB_ORIGIN_AS_FILTERED = ( RSasn, 1101, 10 )
IXP_LC_FILTERED_PREFIX_NOT_IN_ORIGIN_AS = ( RSasn, 1101, 11 )
IXP_LC_FILTERED_RPKI_UNKNOWN = ( RSasn, 1101, 12 )
IXP_LC_FILTERED_RPKI_INVALID = ( RSasn, 1101, 13 )
IXP_LC_FILTERED_TRANSIT_FREE_ASN = ( RSasn, 1101, 14 )
IXP_LC_FILTERED_TOO_MANY_COMMUNITIES = ( RSasn, 1101, 15 )
IXP_LC_INFO_RPKI_VALID = ( RSasn, 1000, 1 )
IXP_LC_INFO_RPKI_UNKNOWN = ( RSasn, 1000, 2 )
IXP_LC_INFO_RPKI_NOT_CHECKED = ( RSasn, 1000, 3 )
IXP_LC_INFO_RPKI_INVALID = ( RSasn, 1000, 4 )
IXP_LC_INFO_IRRDB_INVALID = ( RSasn, 1001, 0 )
IXP_LC_INFO_IRRDB_VALID = ( RSasn, 1001, 1 )
IXP_LC_INFO_IRRDB_NOT_CHECKED = ( RSasn, 1001, 2 )
IXP_LC_INFO_IRRDB_MORE_SPECIFIC = ( RSasn, 1001, 3 )
IXP_LC_INFO_IRRDB_FILTERED_LOOSE = ( RSasn, 1001, 1000 )
IXP_LC_INFO_IRRDB_FILTERED_STRICT = ( RSasn, 1001, 1001 )
IXP_LC_INFO_IRRDB_PREFIX_EMPTY = ( RSasn, 1001, 1002 )
IXP_LC_INFO_FROM_IXROUTESERVER = ( RSasn, 1001, 1100 )
IXP_LC_INFO_SAME_AS_NEXT_HOP = ( RSasn, 1001, 1200 )
from influxdb import InfluxDBClient
from database import *
from large_communities import *
import json
import socket
import requests.packages.urllib3.util.connection as urllib3_cn
import requests
from datetime import datetime
def allowed_gai_family():
family = socket.AF_INET # force IPv4
return family
#FORCE IPV4
urllib3_cn.allowed_gai_family = allowed_gai_family
def expand_community_tuple_to_rest(community_tuple):
assert type(community_tuple) == tuple, "community must be represented by a tuple"
community_size = len(community_tuple)
community_in_slashes_string = "/"
for community_index in range(0,community_size):
# if it is not the last community index
if (community_index!=community_size-1):
community_in_slashes_string = community_in_slashes_string + str(community_tuple[community_index])+"/"
else:
community_in_slashes_string = community_in_slashes_string + str(community_tuple[community_index])
return(community_in_slashes_string)
#print(client)
#ROUTE SERVERS
Route_Servers = ["rsx-vlan505-ipv4.gr-ix.gr","rs1-vlan505-ipv4.gr-ix.gr","rs0-vlan505-ipv4.gr-ix.gr","rs0-vlan400-ipv4.thess.gr-ix.gr","rs1-vlan400-ipv4.thess.gr-ix.gr"]
#REQUESTED PARAMETERS PER RS
number_of_peers_per_rs = {}
number_of_announced_prefixes_per_rs = {}
number_of_irrdb_discards_per_rs = {}
number_of_irrdb_valids_per_rs = {}
number_of_rpki_invalids_per_rs = {}
number_of_irrdb_origin_as_filtered_per_rs = {}
number_of_rpki_valids_per_rs = {}
number_of_rpki_unknown_per_rs = {}
number_of_accepted_prefixes_per_rs = {}
#GET ROUTE SERVER
for RS in Route_Servers:
# initialize temporary counting variables for each RS
announced_prefixes = 0
irrdb_discards = 0
irrdb_valids = 0
irrdb_origin_as_filtered = 0
rpki_valids = 0
rpki_invalids = 0
rpki_unknown = 0
number_of_peers_counter = 0
#GET ROUTE SERVER PROTOCOLS
url = 'http://'+str(RS)+'/api/symbols/protocols'
protocols = requests.get(url=url)
protocols_json = protocols.json()
#tables_json["symbols"][i] -> i is birdseye arithemitc index for each clients table
number_of_bgp_clients = len(protocols_json["symbols"])
number_of_peers_per_rs[RS] = number_of_bgp_clients
for protocol_client_index in range(0,number_of_bgp_clients):
peer = protocols_json["symbols"][protocol_client_index]
#protocol_name corresponds to the peer
# Get only protocols that start with b_, as these correspond to peers
if peer[:2]=="b_":
# Correct peer that starts with b_
# Retrieve announced prefixes per peer
number_of_peers_counter = number_of_peers_counter + 1
announced_prefixes_url = "http://"+str(RS)+"/api/routes/count/protocol/"+str(peer)
announced_prefixes_response = requests.get(url=announced_prefixes_url)
announced_prefixes_response = announced_prefixes_response.json()["count"]["routes"]
announced_prefixes = announced_prefixes + int(announced_prefixes_response)
# Retrieve irrdb_discards per peer
irrdb_discards_url = "http://"+str(RS)+"/api/routes/count/lc-zwild/protocol/"+str(peer)+expand_community_tuple_to_rest(IXP_LC_FILTERED_IRRDB_PREFIX_FILTERED)
irrdb_discards_response = requests.get(url=irrdb_discards_url)
irrdb_discards_response = irrdb_discards_response.json()["count"]["routes"]
irrdb_discards = irrdb_discards + int(irrdb_discards_response)
# Retrieve origin_as_filtered per peer
irrdb_origin_as_filtered_url = "http://"+str(RS)+"/api/routes/count/lc-zwild/protocol/"+str(peer)+expand_community_tuple_to_rest(IXP_LC_FILTERED_IRRDB_ORIGIN_AS_FILTERED)
irrdb_origin_as_filtered_response = requests.get(url=irrdb_origin_as_filtered_url)
irrdb_origin_as_filtered_response = irrdb_origin_as_filtered_response.json()["count"]["routes"]
irrdb_origin_as_filtered = irrdb_origin_as_filtered + int(irrdb_origin_as_filtered_response)
# Retrieve rpki_invalids per peer
rpki_invalids_url = "http://"+str(RS)+"/api/routes/count/lc-zwild/protocol/"+str(peer)+expand_community_tuple_to_rest(IXP_LC_FILTERED_RPKI_INVALID)
rpki_invalids_response = requests.get(url=rpki_invalids_url)
rpki_invalids_response = rpki_invalids_response.json()["count"]["routes"]
rpki_invalids = rpki_invalids + int(rpki_invalids_response)
# These are statistics for routes that are in the master table of the RS, so we do not need to calculate them per peer but only once
rpki_valids_url = "http://"+str(RS)+"/api/routes/count/lc-zwild/table/master"+expand_community_tuple_to_rest(IXP_LC_INFO_RPKI_VALID)
rpki_valids_response = requests.get(url=rpki_valids_url)
rpki_valids_response = rpki_valids_response.json()["count"]["routes"]
rpki_valids =int(rpki_valids_response)
# Attention next to master we do not need a backslash since it is generated by the expand_community_tuple_to_rest
rpki_unknown_url = "http://"+str(RS)+"/api/routes/count/lc-zwild/table/master"+expand_community_tuple_to_rest(IXP_LC_INFO_RPKI_UNKNOWN)
rpki_unknown_response = requests.get(url=rpki_unknown_url)
rpki_unknown_response = rpki_unknown_response.json()["count"]["routes"]
rpki_unknown = int(rpki_unknown_response)
irrdb_valids_url = "http://"+str(RS)+"/api/routes/count/lc-zwild/table/master"+expand_community_tuple_to_rest(IXP_LC_INFO_IRRDB_VALID)
irrdb_valids_response = requests.get(url=irrdb_valids_url)
irrdb_valids_response = irrdb_valids_response.json()["count"]["routes"]
irrdb_valids =int(irrdb_valids_response)
accepted_prefixes_url = announced_prefixes_url = "http://"+str(RS)+"/api/routes/count/table/master"
accepted_prefixes_response = requests.get(url=accepted_prefixes_url)
accepted_prefixes_response = accepted_prefixes_response.json()["count"]["routes"]
accepted_prefixes = accepted_prefixes_response
number_of_peers_per_rs[RS] = number_of_peers_counter
number_of_announced_prefixes_per_rs[RS] = announced_prefixes
number_of_irrdb_discards_per_rs[RS] = irrdb_discards
number_of_irrdb_origin_as_filtered_per_rs[RS] = irrdb_origin_as_filtered
number_of_rpki_invalids_per_rs[RS] = rpki_invalids
number_of_rpki_valids_per_rs[RS] = rpki_valids
number_of_irrdb_valids_per_rs[RS] = irrdb_valids
number_of_rpki_unknown_per_rs[RS] = rpki_unknown
number_of_accepted_prefixes_per_rs[RS] = accepted_prefixes
#print(number_of_announced_prefixes_per_rs)
#print(number_of_irrdb_discards_per_rs)
#print(number_of_rpki_invalids_per_rs)
#print(number_of_rpki_valids_per_rs)
#print(number_of_rpki_unknown_per_rs)
#CONNECT TO THE DATABASE
client = InfluxDBClient(influxdb_ip, infuxdb_port, influxdb_username, influxdb_passwd, influxdb_db)
version = client.ping()
print("Successfully connected to InfluxDB: " + version)
#GET CURRENT TIME
current_time = datetime.utcnow().strftime('%Y-%m-%dT%H:%M:%SZ')
print(current_time)
for RS in Route_Servers:
input_to_influx = [{
"measurement": "route_server_stats",
"tags": {"rs" : str(RS)},
"time": current_time,
"fields": {
"PEERS": number_of_peers_per_rs[RS],
"ANNOUNCED_PREFIXES" : number_of_announced_prefixes_per_rs[RS],
"IRRDB_VALIDS": number_of_irrdb_valids_per_rs[RS],
"IRRDB_PREFIX_FILTERED": number_of_irrdb_discards_per_rs[RS],
"IRRDB_ORIGIN_AS_FILTERED": number_of_irrdb_origin_as_filtered_per_rs[RS],
"RPKI_VALIDS": number_of_rpki_valids_per_rs[RS],
"RPKI_INVALIDS": number_of_rpki_invalids_per_rs[RS],
"RPKI_UNKNOWN": number_of_rpki_unknown_per_rs[RS],
"ACCEPTED_PREFIXES": number_of_accepted_prefixes_per_rs[RS]
}
}]
#print(input_to_influx)
client.write_points(input_to_influx)
from influxdb import InfluxDBClient
from database import *
from large_communities import *
import json
import socket
import requests.packages.urllib3.util.connection as urllib3_cn
import requests
from datetime import datetime
from helpers import *
def allowed_gai_family():
family = socket.AF_INET # force IPv4
return family
#FORCE IPV4
urllib3_cn.allowed_gai_family = allowed_gai_family
def expand_community_tuple_to_rest(community_tuple):
assert type(community_tuple) == tuple, "community must be represented by a tuple"
community_size = len(community_tuple)
community_in_slashes_string = "/"
for community_index in range(0,community_size):
# if it is not the last community index
if (community_index!=community_size-1):
community_in_slashes_string = community_in_slashes_string + str(community_tuple[community_index])+"/"
else:
community_in_slashes_string = community_in_slashes_string + str(community_tuple[community_index])
return(community_in_slashes_string)
selected_communities = [IXP_LC_FILTERED_IRRDB_PREFIX_FILTERED,IXP_LC_FILTERED_IRRDB_ORIGIN_AS_FILTERED,IXP_LC_FILTERED_RPKI_INVALID,IXP_LC_INFO_RPKI_VALID,IXP_LC_INFO_RPKI_UNKNOWN,IXP_LC_INFO_IRRDB_VALID]
class Route_Server_stat:
def __init__(self, rs):
self.rs = rs
self.peers = self.get_peers()
def get_peers(self):
try:
birsdeye_peers_url = 'http://'+str(self.rs)+'/api/symbols/protocols'
protocols = requests.get(url=birsdeye_peers_url)
protocols_json = protocols.json()
number_of_bgp_clients = len(protocols_json["symbols"])
peers = protocols_json["symbols"]
except:
print("Data from birdseye failed to be retrieved")
#tables_json["symbols"][i] -> i is birdseye arithemitc index for each clients table
return(peers)
def get_peers_number(self):
peer_counter = 0
for peer_index in range(0,len(self.peers)):
peer = self.peers[peer_index]
if peer[:2]=="b_":
peer_counter = peer_counter + 1
return(peer_counter)
def get_stat_from_rs_total(self,large_community):
routes_counter = 0
if large_community == None:
requested_large_community_url = "http://"+str(self.rs)+"/api/routes/count/table/master"
else:
requested_large_community_url = "http://"+str(self.rs)+"/api/routes/count/lc-zwild/table/master"+expand_community_tuple_to_rest(large_community)
requested_large_community_response = requests.get(url=requested_large_community_url)
requested_large_community_response = requested_large_community_response.json()["count"]["routes"]
routes_counter = requested_large_community_response
#print(eval(str(large_community)),routes_counter)
return(routes_counter)
def get_stat_from_rs_calculated_per_peer(self,large_community):
routes_counter = 0
for peer_index in range(0,len(self.peers)):
peer = self.peers[peer_index]
if peer[:2]=="b_":
if large_community == None:
requested_large_community_url = "http://"+str(self.rs)+"/api/routes/count/protocol/"+str(peer)
else:
requested_large_community_url = "http://"+str(self.rs)+"/api/routes/count/lc-zwild/protocol/"+str(peer)+expand_community_tuple_to_rest(large_community)
requested_large_community_response = requests.get(url=requested_large_community_url)
requested_large_community_response = requested_large_community_response.json()["count"]["routes"]
routes_counter = routes_counter + int(requested_large_community_response)
#print(eval(str(large_community)),routes_counter)
return(routes_counter)
#return(routes_counter)
'''
#Testing functionality
test_class = Route_Server_stat("rs1-vlan505-ipv4.gr-ix.gr")
test_class.get_peers()
test_class.get_stat_from_rs_total(IXP_LC_FILTERED_RPKI_INVALID)
test_class.get_stat_from_rs_calculated_per_peer(IXP_LC_FILTERED_RPKI_INVALID)
'''
#ROUTE SERVERS
#Route_Servers = ["rsx-vlan505-ipv4.gr-ix.gr","rs1-vlan505-ipv4.gr-ix.gr","rs0-vlan505-ipv4.gr-ix.gr"]
'''
for rs in Route_Servers:
print(rs)
rs_class = Route_Server_stat(rs)
for community in selected_communities:
print("Calculated per peer")
rs_class.get_stat_from_rs_calculated_per_peer(community)
print("Calculated in total")
rs_class.get_stat_from_rs_total(community)
rs_class.get_stat_from_rs_calculated_per_peer(None)
rs_class.get_stat_from_rs_total(None)
'''
#CONNECT TO THE DATABASE
client = InfluxDBClient(influxdb_ip, infuxdb_port, influxdb_username, influxdb_passwd, influxdb_db)
try:
version = client.ping()
if (version):
print("Successfully connected to InfluxDB: " + version)
except Exception, e:
send_email(sender,receivers,"InfluxDB Connection Issue","Python Influx Client could not connect to the database \n"+str(e))
exit()
#selected_communities = [IXP_LC_FILTERED_IRRDB_PREFIX_FILTERED,IXP_LC_FILTERED_IRRDB_ORIGIN_AS_FILTERED,IXP_LC_FILTERED_RPKI_INVALID,IXP_LC_INFO_RPKI_VALID,IXP_LC_INFO_RPKI_UNKNOWN,IXP_LC_INFO_IRRDB_VALID]
#GET CURRENT TIME
current_time = datetime.utcnow().strftime('%Y-%m-%dT%H:%M:%SZ')
print(current_time)
# IPv4 + IPv6 Route Server stats, we handle each RS instance (IPv4, IPv6) as a different RS
Route_Servers = ["rsx-vlan505-ipv4.gr-ix.gr","rs1-vlan505-ipv4.gr-ix.gr","rs0-vlan505-ipv4.gr-ix.gr","rs0-vlan400-ipv4.thess.gr-ix.gr","rs1-vlan400-ipv4.thess.gr-ix.gr"]+["rs1-vlan505-ipv6.gr-ix.gr","rs0-vlan505-ipv6.gr-ix.gr","rs0-vlan400-ipv6.thess.gr-ix.gr","rs1-vlan400-ipv6.thess.gr-ix.gr"]
for rs in Route_Servers:
rs_class = Route_Server_stat(rs)
input_to_influx = [{
"measurement": "route_server_stats",
"tags": {"rs" : str(rs)},
"time": current_time,
"fields": {
"PEERS": rs_class.get_peers_number(),
"ANNOUNCED_PREFIXES" : rs_class.get_stat_from_rs_calculated_per_peer(None),
"IRRDB_VALIDS": rs_class.get_stat_from_rs_total(IXP_LC_INFO_IRRDB_VALID),
"IRRDB_PREFIX_FILTERED": rs_class.get_stat_from_rs_calculated_per_peer(IXP_LC_FILTERED_IRRDB_PREFIX_FILTERED),
"IRRDB_ORIGIN_AS_FILTERED": rs_class.get_stat_from_rs_calculated_per_peer(IXP_LC_FILTERED_IRRDB_ORIGIN_AS_FILTERED),
"RPKI_VALIDS": rs_class.get_stat_from_rs_total(IXP_LC_INFO_RPKI_VALID),
"RPKI_INVALIDS":rs_class.get_stat_from_rs_calculated_per_peer(IXP_LC_FILTERED_RPKI_INVALID),
"RPKI_UNKNOWN": rs_class.get_stat_from_rs_total(IXP_LC_INFO_RPKI_UNKNOWN),
"ACCEPTED_PREFIXES": rs_class.get_stat_from_rs_total(None)
}
}]
#print(input_to_influx)
try:
client.write_points(input_to_influx)
except Exception, e:
send_email(sender,receivers,"InfluxDB Write Issue - InfluxDB service requires restart","Python Influx Client could not write to the database \n"+str(e))
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment