Une MAJ de sécurité est nécessaire sur notre version actuelle. Elle sera effectuée lundi 02/08 entre 12h30 et 13h. L'interruption de service devrait durer quelques minutes (probablement moins de 5 minutes).

Commit 9dcaf50b authored by LETORT Sebastien's avatar LETORT Sebastien Committed by BAIRE Anthony
Browse files

[API]new API to get metrics.

 - It counts things for a webapp on a period of time [from; to [, with a step of aggregation.

3 metrics are proposed :
 - number of jobs per user
 - number of jobs per result
 - number of jobs created
parent 1ebf6602
"""
.. automodule:: api.v1.metrics
:members:
.. automodule:: api.v1.views
:members:
"""
#! /usr/bin/env python3
"""
Metrics API allows to get some metrics about some apps.
* nb_jobs_created
* nb_jobs_per_status
* nb_jobs_per user
They all return an aggregation of the count of jobs on a period of time.
The allgo admin will have access to all app metrics,
whereas a user will have access to its own app metrics.
"""
from datetime import datetime, timedelta
import logging
from django.utils.timezone import make_aware
from django.conf import settings
from django.utils import timezone
from django.db.models import Count
from django.db.models.functions import Trunc
from main.models import Job # , Webapp, JobQueue
log = logging.getLogger('allgo')
# ======================================
class MetricsError(Exception):
"""Generic error generated by this module."""
class ParamError(MetricsError):
"""Error raised for a pb in params,
like its format."""
# ======================================
def __fmt_date(date):
"""date is datetime object (for from/to)"""
return date.date().isoformat()
def str2date(date):
"""convert date to formatted string."""
try:
return datetime.strptime( date, "%Y-%m-%d" )
except ValueError as e:
log.error(e)
raise ParamError(str(e))
__DEFAULT_DELTA = {
'year' : timedelta(days = 5 * 365),
'month': timedelta(days = 2 * 365),
'day' : timedelta(days = 30 + 31),
}
DEFAULT_STEP = 'month'
def __get_params(from_, to, step):
"""get params, if needed, set default or format."""
try:
delta = __DEFAULT_DELTA[step]
except KeyError as e:
msg = "Metrics action {} is unknown.".format( str(e) )
log.error("%s is an invalid step, fall back to default",
e)
step = DEFAULT_STEP
delta = __DEFAULT_DELTA[step]
if to is None:
to = timezone.now()
else:
to = make_aware(str2date(to))
""" ensure that to field is include in the results """
to = to + timezone.timedelta(days=1)
if from_ is None:
from_ = to - __DEFAULT_DELTA[step]
else:
from_ = make_aware(str2date(from_))
return from_, to, step
def __api_dict(from_, to, step, l_records ):
"""from_ and to are datetime.datetime object."""
return {
'data': l_records,
'from': __fmt_date(from_),
'to' : __fmt_date(to),
'step': step,
}
def nb_jobs_per_user(app_id, from_=None, to=None, step=None):
"""compute for the webapp app_id, for each user
the number of jobs "updated" (launched,terminated)
per step period between [from\_ ; to[.
:param app_id: webapp id
:type app_id: int
:param from\_: the beginning of the time interval
:type from\_: string
:param to: the end of the time interval
:type to: string
:param step: data aggregation interval
:type step: string
:return: a dictionnary with 3 keys 'from', 'to' [date] remind the date range,
and 'data' which contains a list of dictionnaries
{user:, uname:, time_period:, n:}
:rtype: dictionnary
"""
from_, to, step = __get_params(from_, to, step)
msg = "provided for nb_jobs_per_user from '{}' to '{}', step of '{}'"\
.format(from_, to, step)
log.info(msg)
counts = Job.objects.filter(webapp=app_id, updated_at__range=[from_, to]) \
.annotate(time_period=Trunc('updated_at', step)) \
.values('time_period', 'user', 'user__username') \
.annotate(n=Count('user'))
# record is a dictionnary built from a sql row response
for record in counts:
record['uname'] = record.pop('user__username')
record['time_period'] = __fmt_date( record.pop('time_period') )
return __api_dict( from_, to, step, list(counts) )
def nb_jobs_per_result(app_id, from_=None, to=None, step=None):
"""compute the number of jobs
for the webapp app_id, for each result (only terminated)
per step period between [from\_ ; to[ (updated_at).
:param app_id: webapp id
:type app_id: int
:param from\_: the beginning of the time interval
:type from\_: string
:param to: the end of the time interval
:type to: string
:param step: data aggregation interval
:type step: string
:return: a dictionnary with 3 keys 'from', 'to' [date] remind the date range,
and 'data' which contains a list of dictionnaries {result:, time_period:, n:}
:rtype: dictionnary
"""
from_, to, step = __get_params(from_, to, step)
msg = "provided for nb_jobs_per_result from '{}' to '{}', step of '{}'"\
.format(from_, to, step)
log.info(msg)
counts = Job.objects.filter(webapp=app_id, updated_at__range=[from_, to]) \
.exclude(result=Job.NONE) \
.annotate(time_period=Trunc('updated_at', step)) \
.values('time_period', 'result' ) \
.annotate(n=Count('result'))
# record is a dictionnary built from a sql row response
for record in counts:
record['result'] = Job.JOB_RESULT_CHOICES[record['result']][1]
record['time_period'] = __fmt_date( record.pop('time_period') )
return __api_dict( from_, to, step, list(counts) )
def nb_jobs_created(app_id, from_=None, to=None, step=None):
"""compute for the webapp app_id, the number of jobs created
per step period between [from\_ ; to[.
:param app_id: webapp id
:type app_id: int
:param from\_: the beginning of the time interval
:type from\_: string
:param to: the end of the time interval
:type to: string
:param step: data aggregation interval
:type step: string
:return: a dictionnary with 3 keys 'from', 'to' [date] remind the date range,
and 'data' which contains a list of dictionnaries {time_period:, n:}
:rtype: dictionnary
"""
from_, to, step = __get_params(from_, to, step)
counts = Job.objects.filter(webapp=app_id, created_at__range=[from_, to]) \
.annotate(time_period=Trunc('created_at', step)) \
.values('time_period') \
.annotate(n=Count('id'))
# record is a dictionnary built from a sql row response
for record in counts:
record['time_period'] = __fmt_date( record.pop('time_period') )
return __api_dict( from_, to, step, list(counts) )
def all_jobs_created(app_id, from_=None, to=None, step=None):
"""compute for the webapp app_id, the number of jobs created
per step period between [from\_ ; to[.
:param app_id: webapp id
:type app_id: int
:param from\_: the beginning of the time interval
:type from\_: string
:param to: the end of the time interval
:type to: string
:param step: data aggregation interval
:type step: string
:return: a dictionnary with 3 keys 'from', 'to' [date] remind the date range,
and 'data' which contains a list of dictionnaries {time_period:, n:}
:rtype: dictionnary
"""
from_, to, step = __get_params(from_, to, step)
msg = "provided from '{}' to '{}', step of '{}'".format(from_, to, step)
log.info(msg)
counts = Job.objects.filter(webapp=app_id, created_at__range=[from_, to]) \
.exclude(result=Job.NONE) \
.annotate(time_period=Trunc('created_at', step)) \
.values('time_period', 'user', 'user__username', 'result') \
.annotate(n=Count('id'))
# record is a dictionnary built from a sql row response
for record in counts:
record['result'] = Job.JOB_RESULT_CHOICES[record['result']][1]
record['user_name'] = record.pop('user__username')
record['time_period'] = __fmt_date( record.pop('time_period') )
return __api_dict( from_, to, step, list(counts) )
# ======================================
ACTIONS = {
"per_user" : nb_jobs_per_user,
"per_result": nb_jobs_per_result,
"created" : nb_jobs_created,
"all" : all_jobs_created,
# ~ "": ,
}
......@@ -6,6 +6,7 @@ app_name = 'api'
urlpatterns = [
url(r'^jobs$', views.jobs, name='jobs'),
url(r'^metrics/(?P<what>\w+)/(?P<app_id>\d+)', views.Metrics.as_view(), name='metrics'),
url(r'^jobs/(?P<pk>\d+)$', views.APIJobView.as_view(), name='job'),
url(r'^datastore/(?P<pk>\d+)/(.*)$', views.APIDownloadView.as_view(), name='download'),
]
......@@ -3,10 +3,14 @@ import os
import config.env
from django.core.validators import ValidationError
from django.core.exceptions import PermissionDenied
from django.http import JsonResponse
from django.urls import reverse
from django.views.decorators.csrf import csrf_exempt
from django.views.generic import View
import api.v1.metrics as metrics
from main.helpers import upload_data, get_base_url, lookup_job_file, get_request_user
from main.mixins import JobAuthMixin
from main.models import Job, Webapp, JobQueue
......@@ -41,6 +45,65 @@ class APIJobView(JobAuthMixin, View):
return JsonResponse({'error': 'Job not found'}, status=404)
class Metrics(View):
"""Metrics view provides only a get access."""
def get(self, request, app_id, what):
"""Method used when accessing the Metrics Api.
:param request: the request object.
:param app_id: the id of the app we want metrics from.
:type app_id: integer
:param what: the kind of metrics we want.
:type what: keyword, key of metrics.ACTIONS
"""
user = get_request_user(request)
if not user:
msg = "API request without http authorisation\n"
msg += "\tagent : %s\n\tadress : %s\n\tquery : %s\n"
log.info(
msg,
request.META.get('HTTP_USER_AGENT', 'agent_unknown'),
request.META.get('REMOTE_ADDR', 'remote_adress_unknown'),
request.META.get('QUERY_STRING', 'query_unknown'),
)
msg = 'API request without http authorisation'
return JsonResponse({'error': msg}, status=401)
try:
msg = "%s is asking for '%s' with app_id '%s'"
log.info(msg, user, what, app_id)
if user.is_superuser:
app = Webapp.objects.get(id=app_id)
else:
app = Webapp.objects.get(id=app_id, user=user)
log.info("app name is *%s*", app.name)
from_ = request.GET.get('from', None)
to = request.GET.get('to', None)
step = request.GET.get('step', None)
msg = "asked from '%s' to '%s', step of '%s'"
log.info(msg, from_, to, step)
# jobs is a dictionnary
jobs = metrics.ACTIONS[what](app_id, from_, to, step)
return JsonResponse({ app.name: jobs })
except Webapp.DoesNotExist as e:
log.error("Webapp not found -%s-", str(e))
return JsonResponse({'error': 'Webapp not found'}, status=404)
except metrics.ParamError as e:
# the error should have already be logged
return JsonResponse({'error': str(e)}, status=400)
except KeyError as e:
log.error("Metrics action %s is unknown.", str(e) )
return JsonResponse({'error': str(e)}, status=404)
@csrf_exempt
def jobs(request):
user = get_request_user(request)
......
......@@ -41,7 +41,7 @@ class BaseModel(models.Model):
This base class overrides .save() to enforce validation of the model
constraints before creating or updating an entry.
The validation is automatically peformed unless `force_insert` or
The validation is automatically performed unless `force_insert` or
`force_update` is true.
see also:
......
......@@ -27,6 +27,9 @@ front-end.
/jobs/<job_id>/issue report an issue
/jobs/<job_id>/relaunch re-run a given job
# same url schema as the API, but here display graphs.
/metrics form to define the graph to display
Todo:
- make the app details available both at /app/<docker_name> (historic
version) and at /apps/<docker_name>/ (more coherent with the overall
......@@ -90,6 +93,8 @@ urlpatterns = [
url(r'^profile/need_validation$', views.UserNeedValidation.as_view(),
name='user_need_validation'),
url(r'^metrics$', views.Metrics.as_view(), name='metrics'),
# Terms of service urls
url(r'^tos$', views.TosDetail.as_view(), name='tos_detail'),
......
......@@ -450,8 +450,9 @@ class WebappImport(ProviderAccessMixin, SuccessMessageMixin, FormView):
current_user = self.request.user
if not current_user.is_superuser:
# get the user EmailAddress that matches the owner of the imported app
email_addr = current_user.email_addresses.filter(
email=js["user"]).first()
email_addr = current_user.email_addresses \
.filter(email=js["user"]) \
.first()
# ensure this app has the same owner
if email_addr is None:
......@@ -1739,6 +1740,30 @@ class RunnerDelete(UserAccessMixin, DeleteView):
return super().delete(request, *args, **kwargs)
# Metrics
# -----------------------------------------------------------------------------
class Metrics(UserAccessMixin, TemplateView):
template_name = 'metrics.html'
def get_context_data(self, **kwargs ):
if self.request.user.is_superuser:
apps = Webapp.objects.all()
else:
apps = Webapp.objects.filter(user=self.request.user)
# context is a dictionnary
context = super().get_context_data(**kwargs)
context['apps'] = apps.order_by('name')
context['show_form'] = 0 != len(apps)
# 'get' returns a AllgoUser object
context['token'] = AllgoUser.objects \
.get(user_id=self.request.user.id) \
.token
return context
@csrf_exempt
def auth(request):
"""
......
This diff is collapsed.
/* AIMS : manage the metrics.html template.
Base class is Metrics.
Its constructor calls _setChartjsData which format the data to be used in Chart object.
Then it defines the build_chart method which build the Chart object.
CreatedPlot inherit from Metrics,
It formats the data (in _setChartjsData) to draw a line chart.
PerResultPlot inherit from Metrics,
It formats the data (in _setChartjsData) to draw a bar chart.
PerUserPlot inherit from Metrics,
It formats the data (in _setChartjsData) to draw a bar chart.
The function draw_plots() is called when the form is submitted.
It queries the API for all metrics, then it build the Metrics object corresponding,
with the data, last it builds the chart.
*/
"use strict";
// ===========================================
function get_random_color()
{
let letters = '0123456789ABCDEF';
let color = '#';
for( let i = 0; i < 6; i++)
color += letters[Math.floor(Math.random() * 16)];
return color;
}
const BLACK = '#000000';
const RED = '#FF0000';
const BLUE = '#0000FF';
function fmt_date(d)
{ return new Date(d).toISOString().substring(0,10); }
function build_list( l_data, key, fn_map )
{
/* build a set of keys from l_data records,
* then return a sorted list of those keys,
* where fn_map is applied on each key.
* It is used to reformat dates.
*/
let d_keys = {};
for( let i=0; i<l_data.length; ++i )
{
let val = l_data[i][key];
d_keys[val] = 1;
}
return Object.keys(d_keys).sort()
.map( x => fn_map(x) );
}
function build_default_dict( l_keys, default_value=0 )
{
let d_defaults = {};
for( let i=0; i<l_keys.length; ++i )
{ let key = l_keys[i]; d_defaults[key] = default_value; }
return d_defaults;
}
// ===========================================
class Metrics
{
constructor( app_name, d_app )
{
// attribute to alter scale of chart. reformat date
this.from = fmt_date(d_app['from']);
this.to = fmt_date(d_app['to']);
this.step = d_app['step'];
this.app_name = app_name;
this.chartjs_data = null;
this.title = null;
this.type = null;
this.d_legend = { position:'top' };
this.chartjs = null;
this._setChartjsData( d_app['data'] );
//~ console.log( this.constructor.name + ": chartjs_data = " + JSON.stringify(this.chartjs_data) );
}
build_chart( id )
{
let ctx = $(id)[0].getContext('2d');
let local_options = {
legend: this.d_legend,
title :
{
display: true,
text: this.title
},
scales:
{
xAxes:
[{
type: 'time',
time:
{
min : this.from,
max : this.to,
unit: this.step,
displayFormats:
{
//~ [this.step]: 'DD-MM-YYYY' //same display whatever the scale
'year' : 'YYYY',
'month': 'MM-YYYY',
'day' : 'DD-MM-YYYY'
}
},
ticks:
{
autoSkip: true
}
}],
yAxes:
[{
display: true,
ticks:
{
beginAtZero: true, // minimum value will be 0.
precision: 0
}
}]
}
};
const options = jQuery.extend( true, {}, local_options, this.options );
this.chartjs = new Chart( ctx,
{
type: this.type,
data: this.chartjs_data,
options: options
});
}
}
class PerUserPlot extends Metrics
{
constructor( app_name, d_app )
{
super( app_name, d_app );
this.title = [
"# jobs per user for " + this.app_name + " app.",
"Time period : " + d_app['from'] + " - " + d_app['to']
];
this.type = 'bar';
this.d_legend.display = false;
this.options = {
scales: {
xAxes: [{ stacked: true }],
yAxes: [{
stacked: true,
ticks: { suggestedMax: this.max_y + 1 }
}]
}
};
}
_setChartjsData( l_data )
{
function __dictionnarize2( l_data, key, l_periods )
{
// turn [ {'time_period':x, key:y, 'n':z }, ...]
// to { y: [{ x: z },{ x: z },...], ... }
// d_default is the default dictionnary {any_key: default_val}
let d_defaults = build_default_dict( l_periods, 0 );
let d_data = {};
let max_n = 0;
for( let i=0; i<l_data.length; ++i )
{
let tp = fmt_date(l_data[i]["time_period"]);
let val = l_data[i][key];
let n = l_data[i]["n"];
if( !d_data[val] )