Commit 23be2204 authored by sebastien letort's avatar sebastien letort

Metrics API. 3 services : nb_jobs_per_user, nb_jobs_per_state, nb_jobs_created.

parent 7e5aca95
#! /usr/bin/env python3
"""
Metrics API allows to get some metrics about some apps.
- the allgo admin will have acces to all app metrics.
- the owner of an app will have access to its metrics.
- a nemo user will have access to nothing.
"""
from datetime import date, datetime
import logging
from django.utils.timezone import make_aware
from django.utils import timezone
from django.db.models import Count, DateTimeField
from django.db.models.functions import TruncHour,TruncMonth, Trunc, Extract
from main.models import Job #, Webapp, JobQueue
log = logging.getLogger('allgo')
# ======================================
def __fmt_date(date):
"""date is datetime object (for from/to)"""
return date.date().isoformat()
DATE_IN_FMT = "%Y-%m-%d"
def str2date(date):
return datetime.strptime( date, DATE_IN_FMT )
DATE_MIN = str2date('2013-01-01')
# SLetort : I used this function to set the default
# because metrics function have all their params, possibly set to None
# and this because i don't want to use the 'request' param from the API
# because those functions will be called also by the interface.
def get_params(from_, to, step, d_defaults):
d_return = d_defaults.copy()
if None is not from_:
d_return['from'] = make_aware(str2date(from_))
if None is not to:
d_return['to'] = make_aware(str2date(to))
if None is not step:
d_return['step'] = step
return ( d_return[x] for x in ('from', 'to', 'step' ))
def __api_dict(from_, to, step, l_records ):
"""from_ and to are datetime.datetime object."""
return {
'data': l_records,
'from': __fmt_date(from_),
'to' : __fmt_date(to),
'step' : step,
}
def nb_jobs_per_user(app_id, from_=None, to=None, step=None):
"""compute for the webapp app_id, for each user the number of jobs "updated" (launched,terminated)
per step period between [from\_ ; to[.
:param app_id: webapp id
:type app_id: int
:param from\_: start of the period studied
:type from\_: string
:param to: end of the period studied
:type to: string
:param step: step period used to aggregate data
:type step: string
:return: a dictionnary with 3 keys 'from', 'to' [date] remind the date range,
and 'data' which contains a list of dictionnaries {user:, uname:, time_period:, n:}
:rtype: dictionnary
"""
d_defaults = { 'from': DATE_MIN, 'to': timezone.now(), 'step': 'month' }
from_,to,step = get_params(from_, to, step, d_defaults)
msg = "provided from '{}' to '{}', step of '{}'"\
.format(from_, to, step)
log.info(msg)
res = Job.objects.filter(webapp=app_id, updated_at__range=[from_,to]) \
.annotate(time_period=Trunc('updated_at', step)) \
.values('time_period', 'user', 'user__username') \
.annotate(n=Count('user'))
# SLETORT: There should be a "django" way of doing this !
for d_res in res:
d_res['uname'] = d_res.pop('user__username')
d_res['time_period'] = __fmt_date( d_res.pop('time_period') )
return __api_dict( from_, to, step, list(res) );
def nb_jobs_per_state(app_id, from_=None, to=None, step=None):
"""compute for the webapp app_id, for each state (result) the number of jobs "updated" (launched,terminated)
per step period between [from\_ ; to[.
:param app_id: webapp id
:type app_id: int
:param from\_: start of the period studied
:type from\_: string
:param to: end of the period studied
:type to: string
:param step: step period used to aggregate data
:type step: string
:return: a dictionnary with 3 keys 'from', 'to' [date] remind the date range,
and 'data' which contains a list of dictionnaries {result:, time_period:, n:}
:rtype: dictionnary
"""
d_defaults = { 'from': DATE_MIN, 'to': timezone.now(), 'step': 'month' }
from_,to,step = get_params(from_, to, step, d_defaults)
msg = "provided from '{}' to '{}', step of '{}'"\
.format(from_, to, step)
log.info(msg)
res = Job.objects.filter(webapp=app_id, updated_at__range=[from_,to]) \
.annotate(time_period=Trunc('updated_at', step)) \
.values('time_period', 'result' ) \
.annotate(n=Count('result'))
for d_res in res:
d_res['result'] = Job.JOB_RESULT_CHOICES[d_res['result']][1]
d_res['time_period'] = __fmt_date( d_res.pop('time_period') )
return __api_dict( from_, to, step, list(res) );
def nb_jobs_created(app_id, from_=None, to=None, step=None):
"""compute for the webapp app_id, the number of jobs created
per step period between [from\_ ; to[.
:param app_id: webapp id
:type app_id: int
:param from\_: start of the period studied
:type from\_: string
:param to: end of the period studied
:type to: string
:param step: step period used to aggregate data
:type step: string
:return: a dictionnary with 3 keys 'from', 'to' [date] remind the date range,
and 'data' which contains a list of dictionnaries {time_period:, n:}
:rtype: dictionnary
"""
d_defaults = { 'from': DATE_MIN, 'to': timezone.now(), 'step': 'month' }
from_,to,step = get_params(from_, to, step, d_defaults)
msg = "provided from '{}' to '{}', step of '{}'"\
.format(from_, to, step)
log.info(msg)
res = Job.objects.filter(webapp=app_id, created_at__range=[from_,to]) \
.annotate(time_period=Trunc('created_at', step)) \
.values('time_period') \
.annotate(n=Count('id'))
for d_res in res:
d_res['time_period'] = __fmt_date( d_res.pop('time_period') )
return __api_dict( from_, to, step, list(res) );
# ======================================
d_actions = {
"per_user" : nb_jobs_per_user,
"per_state": nb_jobs_per_state,
"created" : nb_jobs_created,
# ~ "": ,
}
from django.http import HttpResponse
from django.utils.decorators import method_decorator
from django.views.decorators.csrf import csrf_exempt
"""
Mixins used in API (only).
In Django mixins are used to managed permissions.
"""
import logging
from django.http import JsonResponse
from django.contrib.auth.mixins import UserPassesTestMixin
from main.helpers import get_request_user
from main.models import Job
log = logging.getLogger('allgo')
class ApiAuthMixin(UserPassesTestMixin):
"""API should be accessible uniquely to AllgoUser, ie logged in or with a token.
The user status is checked through get_request_user.
Return a json string with 401 status if unauthorized access.
"""
def test_func(self):
user = get_request_user(self.request)
log.debug( "ApiAuthMixin.test_func user = " + str(user) )
if user is None:
return False
return True
def handle_no_permission(self):
log.debug( "ApiAuthMixin.handle_no_permission" )
# as it should be used only in api, is this test necessary ?
if self.request.path_info.startswith("/api/"):
return JsonResponse({"error": "401 Unauthorized"}, status=401)
return super().handle_no_permission()
......@@ -8,4 +8,5 @@ urlpatterns = [
url(r'^jobs$', views.jobs, name='jobs'),
url(r'^jobs/(?P<pk>\d+)', views.APIJobView.as_view(), name='job'),
url(r'^datastore/(?P<pk>\d+)/(.*)/(.*)', views.APIDownloadView, name='download'),
url(r'^metrics/(?P<what>\w+)/(?P<app_id>\d+)', views.Metrics.as_view(), name='metrics'),
]
import logging
import os
import config.env
from django.core.validators import ValidationError
from django.core.exceptions import PermissionDenied
from django.http import JsonResponse
from django.shortcuts import redirect
from django.views.decorators.csrf import csrf_exempt
from django.views.generic import View
from api.v1.mixins import ApiAuthMixin
import api.v1.metrics as metrics
from main.helpers import upload_data, get_base_url, lookup_job_file, get_request_user
from main.mixins import JobAuthMixin
from main.models import Job, Webapp, JobQueue
......@@ -49,6 +54,60 @@ class APIJobView(JobAuthMixin, View):
return JsonResponse({'error': 'Job not found'}, status=404)
class Metrics(ApiAuthMixin, View):
"""Metrics view provides only a get access."""
def get(self, request, app_id, what):
"""Method used when accessing the Metrics Api.
:param request: the request object.
:param app_id: the id of the app we want metrics from.
:type app_id: integer
:param what: the kind of metrics we want.
:type what: keyword, key of metrics.d_actions
"""
# WARN: here request.user is AnonymousUser, but as been recognised before, in ApiAuthMixin
try:
o_user = get_request_user(request) # IMO doublon avec ApiAuthM
app = Webapp.objects.get(id=app_id) # This is just for debug purpose
msg = "{} is asking for '{}' with app '{}'"\
.format(o_user, what, app.name)
log.info(msg)
if o_user.is_superuser:
o_app = Webapp.objects.filter(id=app_id)
else:
o_app = Webapp.objects.filter(id=app_id, user=o_user)
if 1 != len(o_app):
raise PermissionDenied
from_ = request.GET.get('from', None)
to = request.GET.get('to', None)
step = request.GET.get('step', None)
msg = "asked from '{}' to '{}', step of '{}'"\
.format(from_, to, step)
log.info(msg)
d_jobs = metrics.d_actions[what](app_id, from_, to, step)
return JsonResponse({ app.name: d_jobs })
except Webapp.DoesNotExist as e:
log.error("Webapp not found -%s-", str(e))
return JsonResponse({'error': 'Webapp not found'}, status=404)
except PermissionDenied:
msg = "You are not authorized to access {}"\
.format(app.name)
log.error(msg)
return JsonResponse({'error': 'permission denied'}, status=403)
except KeyError as e:
# WARNING: what if the error comes from another location ?
msg = "Metrics action {} is unknown.".format( str(e) )
log.error(msg)
return JsonResponse({'error': msg}, status=404)
@csrf_exempt
def jobs(request):
user = get_request_user(request)
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment