Metrics site view.

Add ChartJS and momentjs.

django/allgo/main/urls.py

@@ -27,6 +27,9 @@ front-end.
 /jobs/<job_id>/issue        report an issue
 /jobs/<job_id>/relaunch     re-run a given job
 
+# same url schema as the API, but here display graphs.
+/metrics                    form to define the graph to display
+
 Todo:
     - make the app details available both at /app/<docker_name> (historic
       version) and at /apps/<docker_name>/ (more coherent with the overall
@@ -76,6 +79,8 @@ urlpatterns = [
     url(r'^profile/password$', views.UserPasswordUpdate.as_view(), name='user_password'),
     url(r'^profile/need_validation$', views.UserNeedValidation.as_view(), name='user_need_validation'),
 
+    url(r'^metrics$', views.Metrics.as_view(), name='metrics'),
+
     # Terms of service urls
     url(r'^tos$', views.TosDetail.as_view(), name='tos_detail'),
 

django/allgo/main/views.py

@@ -1489,6 +1489,25 @@ class RunnerDelete(UserAccessMixin, DeleteView):
         return super().delete(request, *args, **kwargs)
 
 
+# Metrics
+# -----------------------------------------------------------------------------
+class Metrics(ProviderAccessMixin, TemplateView):
+    template_name = 'metrics.html'
+
+    def get(self, request ):
+        if request.user.is_superuser:
+            l_apps = Webapp.objects.all()
+        else:
+            l_apps = Webapp.objects.filter(user=request.user)
+
+        d_params = {
+                'apps': l_apps.order_by('name'),
+                'show_form': 0 != len(l_apps),
+            }
+        log.info( "Site metrics. User = {}".format(str(request.user)) )
+        return render( request, self.template_name, d_params )
+
+
 @csrf_exempt
 def auth(request):
     """

django/allgo/static/js/metrics.js

+// AIMS  : manage the metrics.html template.
+// NOTE  : nothing yet.
+// AUTHORS : sebastien.letort@irisa.fr
+
+"use strict";
+
+// ===========================================
+function get_random_color()
+{
+    let letters = '0123456789ABCDEF';
+    let color = '#';
+    for( let i = 0; i < 6; i++)
+        color += letters[Math.floor(Math.random() * 16)];
+
+    return color;
+}
+
+const BLACK = '#000000';
+
+function fmt_date(d)
+{ return new Date(d).toISOString().substring(0,10); }
+
+function build_list( l_data, key, fn_map )
+{
+    /* build a set of keys from l_data records,
+     * then return a sorted list of those keys,
+     * where fn_map is applied on each key.
+     * It is used to reformat dates.
+     */
+
+    let d_keys = {};
+    for( let i=0; i<l_data.length; ++i )
+    {
+        let val = l_data[i][key];
+        d_keys[val] = 1;
+    }
+
+    return Object.keys(d_keys).sort()
+                .map( x => fn_map(x) );
+}
+
+function build_default_dict( l_keys, default_value=0 )
+{
+    let d_defaults = {};
+    for( let i=0; i<l_keys.length; ++i )
+    {   let key = l_keys[i]; d_defaults[key] = default_value;   }
+
+    return d_defaults;
+}
+
+// ===========================================
+class Metrics
+{
+    constructor( app_name, d_app )
+    {
+        // attribute to alter scale of chart. reformat date
+        this.from = fmt_date(d_app['from']);
+        this.to   = fmt_date(d_app['to']);
+        this.step = d_app['step'];
+
+        this.app_name = app_name;
+
+        this.chartjs_data = null;
+        this.title = null;
+        this.type  = null;
+
+        this.d_legend = { position:'top' };
+        this.chartjs  = null;
+
+        this._setChartjsData( d_app['data'] );
+        //~ console.log( this.constructor.name + ": chartjs_data = " + JSON.stringify(this.chartjs_data) );
+    }
+
+    build_chart( id )
+    {
+        let ctx = $(id)[0].getContext('2d');
+
+        let local_options = {
+                    legend: this.d_legend,
+                    title :
+                    {
+                        display: true,
+                        text: this.title
+                    },
+                    scales:
+                    {
+                        xAxes:
+                        [{
+                            type: 'time',
+                            time:
+                            {
+                                min : this.from,
+                                max : this.to,
+                                unit: this.step,
+                                displayFormats:
+                                {
+                                    //~ [this.step]: 'DD-MM-YYYY'   //same display whatever the scale
+                                    'year' : 'YYYY',
+                                    'month': 'MM-YYYY',
+                                    'day'  : 'DD-MM-YYYY'
+                                }
+                            },
+                            ticks:
+                            {
+                                autoSkip: true
+                            }
+                        }],
+                        yAxes:
+                        [{
+                            display: true,
+                            ticks:
+                            {
+                                beginAtZero: true,   // minimum value will be 0.
+                                precision: 0
+                            }
+                        }]
+                    }
+                };
+        const options = jQuery.extend( true, {}, local_options, this.options );
+
+        this.chartjs = new Chart( ctx,
+            {
+                type: this.type,
+                data: this.chartjs_data,
+                options: options
+        });
+    }
+}
+
+class PerUserPlot extends Metrics
+{
+    constructor( app_name, d_app )
+    {
+        super( app_name, d_app );
+        this.title = [
+                "# jobs per user for " + this.app_name + " app.",
+                "Time period : " + d_app['from'] + " - " + d_app['to']
+            ];
+        this.type  = 'bar';
+
+        this.d_legend.display = false;
+        this.options = {
+                scales: {
+                        xAxes: [{ stacked: true }],
+                        yAxes: [{ stacked: true }],
+                    }
+            };
+    }
+
+    _setChartjsData( l_data )
+    {
+        function __dictionnarize2( l_data, key, l_periods )
+        {
+            // turn [ {'time_period':x, key:y, 'n':z }, ...]
+            //  to { y: [{ x: z },{ x: z },...], ... }
+            // d_default is the default dictionnary {any_key: default_val}
+            let d_defaults = build_default_dict( l_periods, 0 );
+
+            let d_data = {};
+            for( let i=0; i<l_data.length; ++i )
+            {
+                let tp  = fmt_date(l_data[i]["time_period"]);
+                let val = l_data[i][key];
+                let n   = l_data[i]["n"];
+
+                if( !d_data[val] )
+                    d_data[val] = JSON.parse(JSON.stringify(d_defaults)); // new object.
+
+                d_data[val][tp] = n;
+            }
+
+            return d_data;
+        }
+
+        let l_periods = build_list( l_data, 'time_period', fmt_date );
+
+        // turn [ {'time_period':tp,'uname':u,'n':n }, ...]
+        //  to { u: [{tp: n},{tp: n},, ...], ... }
+        let d_data = __dictionnarize2( l_data, 'uname', l_periods );
+
+        let chartjs_data  = {
+                // keys are dates, and they are strings, in Y-m-d format, so should work.
+                labels  : l_periods,
+                datasets: []
+            };
+
+        for( let user in d_data )
+        {
+            let d_dataset = {
+                    data : l_periods.map( x => d_data[user][x] ),
+                    label: user,
+
+                    backgroundColor: get_random_color(),
+                    borderColor    : BLACK,
+                    //~ borderWidth    : 1,
+                };
+            chartjs_data.datasets.push( d_dataset );
+        }
+
+        this.chartjs_data = chartjs_data;
+    }
+
+}
+
+
+class PerStatePlot extends Metrics
+{
+    constructor( app_name, d_app )
+    {
+        super( app_name, d_app );
+        this.title = [
+                "# jobs per status for " + app_name + " app.",
+                "Time period : " + d_app['from'] + " - " + d_app['to']
+            ];
+        this.type  = 'bar';
+
+        this.d_legend.display = true;
+        this.options = {
+                scales: {
+                        xAxes: [{ stacked: true }],
+                        yAxes: [{ stacked: true }],
+                    }
+            };
+    }
+
+    _setChartjsData( l_data )
+    {
+        function __dictionnarize( l_data, key, l_periods, l_states )
+        {
+            // turn [ {'time_period':x, key:y, 'n':z }, ...]
+            //  to { y: { x: z }, ... }
+            // d_defaults is the default dictionnary {any_key: default_val}
+            let d_defaults = build_default_dict( l_periods, 0 );
+
+            let d_data = {};
+            for( let i=0; i<l_states.length; ++i )
+            {
+               d_data[l_states[i]] = JSON.parse(JSON.stringify(d_defaults)); // new object.
+            }
+
+
+            for( let i=0; i<l_data.length; ++i )
+            {
+                let tp  = fmt_date( l_data[i]["time_period"] );
+                let val = l_data[i][key];
+                let n   = l_data[i]["n"];
+
+                d_data[val][tp] = n;
+            }
+
+            return d_data;
+        }
+
+        // SLETORT: Note : I really don't like the way I had to write 3 times the labels !
+        let d_colors   = {
+            "NONE"   : '#000000',
+            "SUCCESS": '#00FF00',
+            "ERROR"  : '#FF0000',
+            "ABORTED": '#FF8000',
+            "TIMEOUT": '#DDAA22'
+        };
+
+        let l_periods = build_list( l_data, 'time_period', fmt_date );
+        let l_states  = [ "NONE", "SUCCESS", "ERROR", "ABORTED", "TIMEOUT" ]; // to ensure the order.
+
+        let d_data = __dictionnarize( l_data, 'result', l_periods, l_states );
+
+        // now let's turn this dict to chartjs_data
+        let chartjs_data = { labels: l_periods, datasets: [] };
+        for( let i=0; i<l_states.length; ++i )
+        {
+            let status = l_states[i];
+
+            chartjs_data.datasets[i] =
+                {
+                    data : l_periods.map( x => d_data[status][x] ),
+                    label: status.toLowerCase(),
+
+                    backgroundColor: d_colors[status],
+                    borderColor    : d_colors[status],
+                    borderWidth    : 1,
+
+                    fill : false,
+                };
+        }
+
+        this.chartjs_data = chartjs_data;
+    }
+}
+
+
+class CreatedPlot extends Metrics
+{
+    constructor( app_name, d_app )
+    {
+        super( app_name, d_app );
+        this.title = [
+                "# jobs created for " + app_name + " app.",
+                "Time period : " + d_app['from'] + " - " + d_app['to']
+            ];
+        this.type  = 'line';
+        this.d_legend.display = false;
+        this.options = {
+                lineTension: 0
+            };
+
+    }
+
+    _setChartjsData( l_data )
+    {
+        let l_periods = build_list( l_data, 'time_period', fmt_date );
+
+        let d_data = {}
+        for( let i=0; i<l_data.length; ++i )
+        {
+            let tp  = fmt_date( l_data[i].time_period );
+            let val = l_data[i].n;
+            d_data[tp] = val; // by construction there should be no collision.
+        }
+
+        let chartjs_data = {
+                labels: l_periods,
+                datasets: [ { data:[], lineTension: 0 } ]
+            };
+
+        for( let i=0; i<l_periods.length; ++i )
+        {
+            let tp = l_periods[i];
+            chartjs_data.datasets[0]['data'].push( d_data[tp] );
+        }
+
+        this.chartjs_data = chartjs_data;
+    }
+}
+
+// ===========================================
+// ===========================================
+// ===========================================
+// Note: I need to store Chart object to destroy them when redrawing
+//  cf: https://stackoverflow.com/questions/40056555/destroy-chart-js-bar-graph-to-redraw-other-graph-in-same-canvas
+//  cf: https://stackoverflow.com/questions/24815851/how-to-clear-a-chart-from-a-canvas-so-that-hover-events-cannot-be-triggered
+// Note: another solution could be to update chart if they exist.
+const D_CHARTS = {
+    // api keyword : [ html elt id, js class, object ]
+    'per_user' : [ '#per_user_plot', PerUserPlot, null ],
+    'per_state': [ '#per_state_plot', PerStatePlot, null ],
+    'created' : [ '#created_plot', CreatedPlot, null ],
+};
+
+function build_API_url( which_chart )
+{
+    // get valuable info from form
+    let app_id      = $('#app').val();
+    let step        = $('#step').val();
+    let from        = $('#from').val();
+    let to          = $('#to').val();
+
+    // build the API URL
+    let api_url  = "api/v1/metrics"
+                    + "/" + which_chart
+                    + "/" + app_id
+                    + "?step=" + step;
+    if( from )
+        api_url += "&from=" + from;
+    if( to )
+        api_url += "&to=" + to;
+    console.log( "API URL : " + api_url );
+
+    return api_url;
+}
+
+function draw_plots()
+{
+    for( let which_chart in D_CHARTS )
+    {
+        let API_URL = build_API_url( which_chart );
+        fetch(API_URL).then(function(response)
+        {
+            if( !response.ok )
+            {
+                let msg = "There was a problem querying the API.\n";
+                msg += "reponse status : " + response.status;
+                msg += " (" + response.statusText + ").";
+                console.log(msg);
+                //~ $("#metric_plot").value = "Problem with the API. Can't get the data.";
+            }
+            else
+            {
+                response.json().then(function(json)
+                {
+                    // code for ChartJS
+                    for( let app in json )
+                    {
+                        let [id,cstr,obj] = D_CHARTS[which_chart];
+                        if( obj )
+                            obj.destroy();
+                        //~ console.log( id );
+
+                        let o_plot = new cstr( app, json[app] );
+                        o_plot.build_chart( id );
+                        D_CHARTS[which_chart][2] = o_plot.chartjs;
+                    }
+                },
+                function( reason )
+                {
+                    // error in response.json() promise
+                    console.log(reason);
+                });
+            }
+        });
+    } // for which_chart
+
+    return false;
+}

django/allgo/templates/metrics.html

+{% extends "base.html" %}
+{% load static converters humanize %}
+
+{% block title %}A||GO | Metrics{% endblock %}
+{% block breadcrumb %}
+    <li class="breadcrumb-item active" aria-current="page">Metrics</li>
+{% endblock %}
+
+{% block content %}
+    {% if show_form == False %}
+    <p class="container">You do not have access to any Webapp, so nothing to ask for.</p>
+    {% else %}
+        <form id="metrics_form" onsubmit="return draw_plots()">
+            <fieldset>
+              <label for="app">App</label>
+              <select id="app">
+                {% for app in apps %}
+                    <option value="{{app.id}}">{{ app.name }}</option>
+                {% endfor %}
+              </select>
+
+    <!--
+              <label for="chart">chart</label>
+              <select id="chart">
+                  <option value="per_user">#jobs per user</option>
+                  <option value="launched">#jobs running</option>
+                  <option value="per_res">#jobs by state and period</option>
+              </select>
+    -->
+
+              <label for="from">From</label>
+              <input id="from" type="date" min="2015-01-01" />
+
+              <label for="to">To</label>
+              <input id="to" type="date" />
+
+              <label for="step">group by</label>
+              <select id="step">
+                <option>year</option>
+                <option>month</option>
+<!--
+                <option>week</option>
+-->
+                <option>day</option>
+<!--
+                <option>hour</option>
+-->
+              </select>
+            </fieldset>
+            <input type="submit" />
+            <canvas id="created_plot" width="400px" height="200px"></canvas>
+            <canvas id="per_user_plot" width="400px" height="200px"></canvas>
+            <canvas id="per_state_plot" width="400px" height="200px"></canvas>
+        </form>
+    {% endif %}
+{% endblock %}
+
+{% block javascript %}
+    {{ block.super }}
+    <script src="{% static 'js/bootstrap.min.js' %}"></script>
+    <script src="{% static 'js/moment.min.js' %}"></script>
+    <script src="{% static 'js/Chart.min.js' %}"></script>
+    <script src="{% static 'js/metrics.js' %}"></script>
+
+    <script src="{% static 'js/jquery-3.3.1.slim.min.js' %}"></script>
+
+{% endblock %}

django/allgo/templates/partials/_header.html

@@ -48,6 +48,9 @@
             <a class="nav-link" href="{% url 'main:runner_list' %}"><i class="fab fa-hubspot"></i> Runners</a>
           </li>
           {% endcomment %}
+          <li  class="nav-item {% is_active request 'metrics' %}">
+            <a class="nav-link" href="{% url 'main:metrics' %}"><i class="fas fa-chart-bar"></i> Metrics</a>
+          </li>
           <li class="nav-item {% is_active request 'user_detail' 'user_token' 'user_ssh_add' 'user_ssh_delete' 'user_password' %}">
             <a class="nav-link" href="{% url 'main:user_detail' %}"><i class="fas fa-user"></i> Profile</a>
           </li>