94 lines
3.7 KiB
Python
94 lines
3.7 KiB
Python
|
|
import datetime
|
|
import logging
|
|
import os
|
|
import requests
|
|
|
|
try:
|
|
from alerta.plugins import app # alerta >= 5.0
|
|
except ImportError:
|
|
from alerta.app import app # alerta < 5.0
|
|
from alerta.plugins import PluginBase
|
|
|
|
LOG = logging.getLogger('alerta.plugins.prometheus')
|
|
|
|
DEFAULT_ALERTMANAGER_API_URL = 'http://localhost:9093'
|
|
|
|
ALERTMANAGER_API_URL = os.environ.get('ALERTMANAGER_API_URL') or app.config.get('ALERTMANAGER_API_URL', None)
|
|
ALERTMANAGER_API_KEY = os.environ.get('ALERTMANAGER_API_KEY') or app.config.get('ALERTMANAGER_API_KEY', '') # not used
|
|
ALERTMANAGER_SILENCE_DAYS = os.environ.get('ALERTMANAGER_SILENCE_DAYS') or app.config.get('ALERTMANAGER_SILENCE_DAYS', 1)
|
|
|
|
|
|
class AlertmanagerSilence(PluginBase):
|
|
|
|
def pre_receive(self, alert):
|
|
return alert
|
|
|
|
def post_receive(self, alert):
|
|
return
|
|
|
|
def status_change(self, alert, status, text):
|
|
|
|
if alert.event_type != 'prometheusAlert':
|
|
return
|
|
|
|
if alert.status == status:
|
|
return
|
|
|
|
if status == 'ack':
|
|
LOG.debug('Alertmanager: Add silence for alertname=%s instance=%s', alert.event, alert.resource)
|
|
data = {
|
|
"matchers": [
|
|
{
|
|
"name": "alertname",
|
|
"value": alert.event
|
|
},
|
|
{
|
|
"name": "instance",
|
|
"value": alert.resource
|
|
}
|
|
],
|
|
"startsAt": datetime.datetime.utcnow().replace(microsecond=0).isoformat() + ".000Z",
|
|
"endsAt": (datetime.datetime.utcnow() + datetime.timedelta(days=ALERTMANAGER_SILENCE_DAYS))
|
|
.replace(microsecond=0).isoformat() + ".000Z",
|
|
"createdBy": "alerta",
|
|
"comment": text if text != '' else "silenced by alerta"
|
|
}
|
|
|
|
base_url = ALERTMANAGER_API_URL or alert.attributes.get('externalUrl', DEFAULT_ALERTMANAGER_API_URL)
|
|
url = base_url + '/api/v1/silences'
|
|
try:
|
|
r = requests.post(url, json=data, timeout=2)
|
|
except Exception as e:
|
|
raise RuntimeError("Alertmanager: ERROR - %s", e)
|
|
LOG.debug('Alertmanager: %s - %s', r.status_code, r.text)
|
|
|
|
# example r={"status":"success","data":{"silenceId":8}}
|
|
try:
|
|
silenceId = r.json()['data']['silenceId']
|
|
alert.attributes['silenceId'] = silenceId
|
|
text = text + ' (silenced in Alertmanager)'
|
|
except Exception as e:
|
|
raise RuntimeError("Alertmanager: ERROR - %s", e)
|
|
LOG.debug('Alertmanager: Added silenceId %s to attributes', silenceId)
|
|
|
|
elif status == 'open':
|
|
LOG.debug('Alertmanager: Remove silence for alertname=%s instance=%s', alert.event, alert.resource)
|
|
|
|
silenceId = alert.attributes.get('silenceId', None)
|
|
if silenceId:
|
|
base_url = ALERTMANAGER_API_URL or alert.attributes.get('externalUrl', DEFAULT_ALERTMANAGER_API_URL)
|
|
url = base_url + '/api/v1/silence/%s' % silenceId
|
|
try:
|
|
r = requests.delete(url, timeout=2)
|
|
except Exception as e:
|
|
raise RuntimeError("Alertmanager: ERROR - %s", e)
|
|
LOG.debug('Alertmanager: %s - %s', r.status_code, r.text)
|
|
|
|
try:
|
|
alert.attributes['silenceId'] = None
|
|
except Exception as e:
|
|
raise RuntimeError("Alertmanager: ERROR - %s", e)
|
|
LOG.debug('Alertmanager: Removed silenceId %s from attributes', silenceId)
|
|
|
|
return alert, status, text
|