Packaging Vidjil server

Added automation scripts for building the vidjil server Debian package

packaging/build-server.sh

+#!/bin/bash
+
+. ./build-generic.sh
+
+MY_PWD="$PWD"
+
+source_files="../browser ../data ../doc ../README.org ../reports ../server ../tools"
+dest_files="server/vidjil-server"
+exec_dir="server"
+
+copy_files -s="$source_files" -d="$dest_files"
+
+cd "$MY_PWD/$dest_files/germline"
+make
+cd "$MY_PWD/$dest_files/data"
+
+cd "$MY_PWD/$exec_dir"
+bash "$MY_PWD/mkdeb"
+
+remove_files -s="$source_files" -d="$dest_files"
+exit 0

packaging/server/conf/apache_web2py

+WSGIDaemonProcess web2py user=www-data group=www-data processes=1 threads=1
+
+<VirtualHost *:80>
+
+    DocumentRoot /var/www
+    <Directory />
+      Options FollowSymLinks
+      AllowOverride None
+    </Directory>
+
+    <Directory /var/www/>
+      Options Indexes FollowSymLinks MultiViews
+      AllowOverride all
+      Order allow,deny
+      allow from all
+    </Directory>
+
+    ScriptAlias /cgi/ /usr/lib/cgi-bin/
+
+    <Directory /usr/lib/cgi-bin/>
+      Options Indexes FollowSymLinks
+      Options +ExecCGI
+      #AllowOverride None
+      Require all granted
+      AddHandler cgi-script cgi pl
+    </Directory>
+
+  <Directory /usr/share/vidjil/browser>
+    AllowOverride None
+  </Directory>
+
+  CustomLog /var/log/apache2/access.log common
+  ErrorLog /var/log/apache2/error.log
+</VirtualHost>
+
+<VirtualHost *:443>
+  SSLEngine on
+  SSLCertificateFile /etc/apache2/ssl/self_signed.cert
+  SSLCertificateKeyFile /etc/apache2/ssl/self_signed.key
+
+  WSGIProcessGroup web2py
+  WSGIScriptAlias / /usr/share/vidjil/server/web2py/wsgihandler.py
+  WSGIPassAuthorization On
+
+  <Directory /usr/share/vidjil/server/web2py>
+    AllowOverride None
+    Require all denied
+    <Files wsgihandler.py>
+      Require all granted
+    </Files>
+  </Directory>
+
+  <Directory /usr/share/vidjil/browser>
+    AllowOverride None
+  </Directory>
+
+  AliasMatch ^/([^/]+)/static/(?:_[\d]+.[\d]+.[\d]+/)?(.*) \\
+        /usr/share/vidjil/server/web2py/applications/\$1/static/\$2
+
+  <Directory /usr/share/vidjil/server/web2py/applications/*/static/>
+    Options -Indexes
+    ExpiresActive On
+    ExpiresDefault \"access plus 1 hour\"
+    Require all granted
+  </Directory>
+
+  CustomLog /var/log/apache2/ssl-access.log common
+  ErrorLog /var/log/apache2/error.log
+</VirtualHost> 

packaging/server/conf/conf.js

+/* 
+ * Vidjil browser, main configuration file
+ * This file must be named 'js/conf.js' to be taken into account
+ * */
+
+var config = {
+
+    /* Used for the 'align' script 
+     * If this is not defined, the 'align' button is not available
+     */    
+    "cgi_address" : "default", // Public test server
+    // "cgi_address" : "http://127.0.1.1/cgi-bin/",
+
+    /* The following options control how the user may have access to .vidjil files.
+     * Any combination of 1), 2) and 3) should work
+     */
+
+    /* 1) Patient database */
+    "use_database" : true,
+    "db_address" : "default",
+
+    /* 2) and 3) Static files
+    /* - relative paths if Vidjil browser is online on the same server
+     * - absolute paths to a CORS active server only if browser is offline or on another server 
+     */
+
+    /* 2) Menu with a list of static files */
+    "file_menu" : {
+        "path" : "/data/",
+        "file" : [
+            "Stanford_S22.fasta"
+        ]
+    },
+    
+    /* 3) Static file autoload, possibly with an .analysis file */
+    "autoload" : "data/Stanford-S22.vidjil"
+    // "autoload_analysis" : "data/Stanford-S22.analysis"
+}

packaging/server/conf/defs.py

+###
+### Vidjil server, main configuration file
+### This file must be named 'modules/defs.py' to be taken into account
+###
+
+### Email notifications for server errors
+SMTP_SERVER = 'localhost'
+# SMTP_SERVER = 'logging' # no mail
+FROM_EMAIL = 'srv@example.org'
+ADMIN_EMAILS = ['notifications@example.org']
+
+### address for the sql database
+###
+DB_ADDRESS = 'sqlite://storage.sqlite'
+DB_ADDRESS = 'mysql://vidjil:XXXXXXXX@localhost/vidjil'
+
+### Backup file
+DB_BACKUP_FILE = 'backup.csv'
+
+### Upload directory for .fasta/.fastq.
+### Old sequences files could be thrown away.
+### No regular backup.
+
+DIR_SEQUENCES = '/mnt/upload/uploads/'
+
+### Upload directory for .vidjil/.fused/.analysis
+### Regularly backuped
+
+DIR_RESULTS = '/mnt/result/results/'
+
+### Temporary directory to store vidjil results, and basename of files in this directory
+### Formatted with 'data_id' in models/task.py
+DIR_OUT_VIDJIL_ID = '/mnt/result/tmp/out-%06d/'
+DIR_PRE_VIDJIL_ID = '/mnt/result/tmp/pre/out-%06d/'
+BASENAME_OUT_VIDJIL_ID = '%06d'
+
+### Directory for program used in task.py
+### relative path start from server/web2py
+DIR_VIDJIL = '/usr/bin/vidjil'
+DIR_FUSE = '../../tools'
+DIR_MIXCR = '../../'
+### Port on which to run the fuse server
+### Used in models/task.py and in /server/fuse_server.py
+PORT_FUSE_SERVER = 12789
+
+### Log files
+DIR_LOG = '/var/vidjil/'
+LOG_INFO = DIR_LOG + 'vidjil.log'
+LOG_DEBUG = DIR_LOG + 'vidjil-debug.log'
+
+### Timeouts
+# Delay in seconds before a task is considered in timeout
+TASK_TIMEOUT = 2 * 60 * 60
+
+### Reverse IP file
+REVERSE_IP = '/home/vidjil/ips.txt'
+
+### Locus (should be parsed from germlines.data)
+LOCUS = ['TRA', 'TRA+D', 'TRB', 'TRG', 'TRD', 'TRD+',
+         'IGH', 'IGH+', 'IGK', 'IGK+', 'IGL']

packaging/server/conf/init_fuse_server.conf

+description \"fuse server vidjil\"
+start on (local-filesystems and net-device-up IFACE=eth0)
+stop on shutdown
+respawn
+respawn limit 8 60 # Give up if restart occurs 8 times in 60 seconds.
+chdir /usr/share/vidjil/server/
+exec  sudo -u www-data python fuse_server.py

packaging/server/conf/init_web2py_scheduler.conf

+description \"web2py vidjil task scheduler\"
+start on (local-filesystems and net-device-up IFACE=eth0)
+stop on shutdown
+respawn
+respawn limit 8 60 # Give up if restart occurs 8 times in 60 seconds.
+exec  sudo -u www-data python /usr/share/vidjil/server/web2py/web2py.py -K vidjil,vidjil,vidjil

packaging/server/conf/nginx_gzip.conf

+gzip on;
+gzip_disable "msie6";
+gzip_vary on;
+gzip_proxied any;
+gzip_comp_level 6;
+gzip_buffers 16 8k;
+gzip_http_version 1.1;
+gzip_types text/plain text/css application/json application/x-javascript text/xml application/xml application/xml+rss text/javascript;

packaging/server/conf/nginx_gzip_static.conf

+gzip_static on;
+gzip_http_version   1.1;
+gzip_proxied        expired no-cache no-store private auth;
+gzip_disable        "MSIE [1-6]\.";
+gzip_vary           on;

packaging/server/conf/nginx_web2py

+server {
+    listen          80;
+    server_name     \$hostname;
+
+    location /browser {
+        root /var/www;
+        expires 1h;
+
+        error_page 405 = $uri;
+    
+    }
+
+    location /germline {
+        root /usr/share/vidjil/;
+        expires 1h;
+
+        error_page 405 = $uri;
+    
+    }
+
+    ###to enable correct use of response.static_version
+    #location ~* ^/(\w+)/static(?:/_[\d]+\.[\d]+\.[\d]+)?/(.*)$ {
+        #    alias /usr/share/vidjil/server/web2py/applications/\$1/static/\$2;
+        #    expires max;
+        #
+    #}
+    ###
+
+    location ~* ^/(\w+)/static/ {
+        root /usr/share/vidjil/server/web2py/applications/;
+        expires max;
+        ### if you want to use pre-gzipped static files (recommended)
+        ### check scripts/zip_static_files.py and remove the comments
+        # include /etc/nginx/conf.d/web2py/gzip_static.conf;
+        ###
+        
+    }
+    location / {
+        #uwsgi_pass      127.0.0.1:9001;
+        uwsgi_pass      unix:///tmp/web2py.socket;
+        include         uwsgi_params;
+        uwsgi_param     UWSGI_SCHEME \$scheme;
+        uwsgi_param     SERVER_SOFTWARE    nginx/\$nginx_version;
+
+        proxy_read_timeout 600;
+        ###remove the comments to turn on if you want gzip compression of your pages
+        include /etc/nginx/conf.d/web2py/gzip.conf;
+        ### end gzip section
+
+        client_max_body_size 20G;
+    }
+
+    client_max_body_size 20G;
+
+    location /cgi/ {
+        gzip off;
+        root  /usr/share/vidjil/browser/;
+        # Fastcgi socket
+        fastcgi_pass  unix:/var/run/fcgiwrap.socket;
+        # Fastcgi parameters, include the standard ones
+        include /etc/nginx/fastcgi_params;
+        # Adjust non standard parameters (SCRIPT_FILENAME)
+        fastcgi_param SCRIPT_FILENAME  \$document_root\$fastcgi_script_name;
+    
+    }
+
+}
+server {
+        listen 443 default_server ssl;
+        server_name     \$hostname;
+        ssl_certificate         /etc/nginx/ssl/web2py.crt;
+        ssl_certificate_key     /etc/nginx/ssl/web2py.key;
+        ssl_prefer_server_ciphers on;
+        ssl_session_cache shared:SSL:10m;
+        ssl_session_timeout 10m;
+        ssl_ciphers ECDHE-RSA-AES256-SHA:DHE-RSA-AES256-SHA:DHE-DSS-AES256-SHA:DHE-RSA-AES128-SHA:DHE-DSS-AES128-SHA;
+        ssl_protocols TLSv1 TLSv1.1 TLSv1.2;
+        keepalive_timeout    70;
+        location / {
+            #uwsgi_pass      127.0.0.1:9001;
+            uwsgi_pass      unix:///tmp/web2py.socket;
+            include         uwsgi_params;
+            uwsgi_param     UWSGI_SCHEME \$scheme;
+            uwsgi_param     SERVER_SOFTWARE    nginx/\$nginx_version;
+            ###remove the comments to turn on if you want gzip compression of your pages
+            # include /etc/nginx/conf.d/web2py/gzip.conf;
+            ### end gzip section
+
+            proxy_read_timeout 600;
+            client_max_body_size 20G;
+            ###
+        
+        }
+        ## if you serve static files through https, copy here the section
+        ## from the previous server instance to manage static files
+
+}

packaging/server/conf/systemd_fuse_server.service

+[Unit]
+Description=vidjil fuse server
+After=network-online.target local-fs.target
+
+[Service]
+WorkingDirectory=/usr/share/vidjil/server
+ExecStart=/usr/bin/python /usr/share/vidjil/server/fuse_server.py
+User=www-data
+Restart=always
+KillSignal=SIGQUIT
+Type=simple
+StandardError=syslog
+NotifyAccess=all
+
+[Install]
+WantedBy=multi-user.target

packaging/server/conf/systemd_web2py_scheduler.service

+[Unit]
+Description=web2py vidjil task scheduler
+After=network-online.target
+
+[Service]
+ExecStart=/usr/bin/python /usr/share/vidjil/server/web2py/web2py.py -K vidjil,vidjil,vidjil
+Type=simple
+
+[Install]
+WantedBy=multi-user.target

packaging/server/conf/uwsgi.ini

+[uwsgi]
+
+socket = /tmp/web2py.socket
+pythonpath = /usr/share/vidjil/server/web2py/
+mount = /=wsgihandler:application
+processes = 4
+master = true
+harakiri = 1200
+reload-mercy = 8
+cpu-affinity = 1
+stats = /tmp/stats.socket
+max-requests = 2000
+limit-as = 512
+reload-on-as = 256
+reload-on-rss = 192
+touch-reload = /usr/share/vidjil/server/web2py/applications/vidjil/modules/defs.py
+uid = www-data
+gid = www-data
+cron = 0 0 -1 -1 -1 python /usr/share/vidjil/server/web2py/web2py.py -Q -S welcome -M -R scripts/sessions2trash.py -A -o
+no-orphans = true
+ignore-sigpipe = true

packaging/server/conf/wsgihandler.py

+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+
+"""
+This file is part of the web2py Web Framework
+Copyrighted by Massimo Di Pierro <mdipierro@cs.depaul.edu>
+License: LGPLv3 (http://www.gnu.org/licenses/lgpl.html)
+
+
+This is a WSGI handler for Apache
+Requires apache+mod_wsgi.
+
+In httpd.conf put something like:
+
+    LoadModule wsgi_module modules/mod_wsgi.so
+    WSGIScriptAlias / /path/to/wsgihandler.py
+
+"""
+
+# change these parameters as required
+LOGGING = False
+SOFTCRON = False
+
+import sys
+import os
+
+path = os.path.dirname(os.path.abspath(__file__))
+os.chdir(path)
+
+if not os.path.isdir('applications'):
+    raise RuntimeError('Running from the wrong folder')
+
+sys.path = [path] + [p for p in sys.path if not p == path]
+
+sys.stdout = sys.stderr
+
+import gluon.main
+
+if LOGGING:
+    application = gluon.main.appfactory(wsgiapp=gluon.main.wsgibase,
+                                        logfilename='httpserver.log',
+                                        profiler_dir=None)
+else:
+    application = gluon.main.wsgibase
+
+if SOFTCRON:
+    from gluon.settings import global_settings
+    global_settings.web2py_crontype = 'soft'

packaging/server/debian/changelog

+vidjil-server (2016.08.12) stable; urgency=medium
+
+  * Added missing dependencies
+
+ -- Vidjil Team <team@vidjil.org>  Fri, 12 Aug 2016 15:14:02 +0200

packaging/server/debian/compat

+9

packaging/server/debian/control

+Source: vidjil-server
+Section: misc
+Priority: optional
+Maintainer: Vidjil Team <team@vidjil.org>
+Build-Depends: debhelper (>=9), dh-python, dh-systemd (>= 1.5), python-all-dev
+Standards-Version: 3.9.7
+Homepage: http://www.vidjil.org
+#Vcs-Git: git://anonscm.debian.org/collab-maint/vidjil-server.git
+#Vcs-Browser: https://anonscm.debian.org/cgit/collab-maint/vidjil-server.git
+
+Package: vidjil-server
+Architecture: all
+Depends: vidjil, vidjil-germline, ssh, zip, unzip, tar (>=1.2), openssh-server, python,
+ ipython, python-enum34, postfix, wget, python-matplotlib, python-reportlab, build-essential,
+ uwsgi, uwsgi-plugin-python, python-requests, python-web2py, ${misc:Depends}
+Recommends: apache2, nginx-full, init, systemd
+Description: Vidjil visualisation server
+ A web2py-based webserver that utilises nginx to provide visualisation
+ tools and data storage for the vidjil application

packaging/server/debian/copyright

+Format: https://www.debian.org/doc/packaging-manuals/copyright-format/1.0/
+Upstream-Name: vidjil-server
+Source: http://vidjil.org
+
+Files: *
+Copyright: 2016 Vidjil Team <team@vidjil.org>
+License: GPL-3.0+
+
+Files: debian/*
+Copyright: 2016 Vidjil Team <team@vidjil.org>
+License: GPL-3.0+
+
+License: GPL-3.0+
+ This program is free software: you can redistribute it and/or modify
+ it under the terms of the GNU General Public License as published by
+ the Free Software Foundation, either version 3 of the License, or
+ (at your option) any later version.
+ .
+ This package is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ GNU General Public License for more details.
+ .
+ You should have received a copy of the GNU General Public License
+ along with this program. If not, see <https://www.gnu.org/licenses/>.
+ .
+ On Debian systems, the complete text of the GNU General
+ Public License version 3 can be found in "/usr/share/common-licenses/GPL-3".

packaging/server/debian/install

+vidjil-server/* usr/share/vidjil
+conf/* etc/vidjil

packaging/server/debian/rules

+#!/usr/bin/make -f
+# See debhelper(7) (uncomment to enable)
+# output every command that modifies files on the build system.
+#export DH_VERBOSE = 1
+
+
+# see FEATURE AREAS in dpkg-buildflags(1)
+#export DEB_BUILD_MAINT_OPTIONS = hardening=+all
+
+# see ENVIRONMENT in dpkg-buildflags(1)
+# package maintainers to append CFLAGS
+#export DEB_CFLAGS_MAINT_APPEND  = -Wall -pedantic
+# package maintainers to append LDFLAGS
+#export DEB_LDFLAGS_MAINT_APPEND = -Wl,--as-needed
+
+
+%:
+	dh $@ --with python2 --with systemd	
+
+
+# dh_make generated override targets
+# This is example for Cmake (See https://bugs.debian.org/641051 )
+#override_dh_auto_configure:
+#	dh_auto_configure -- #	-DCMAKE_LIBRARY_PATH=$(DEB_HOST_MULTIARCH)
+

packaging/server/debian/source/format

+3.0 (native)

packaging/server/debian/templates

+Template: vidjil/reconfigure-webserver
+Type: select
+Choices: apache2, nginx
+Description: Web server to configure automatically
+ Any webserver can be configured to serve the vidjil webapp,
+ however we only supply files for Apache2 and Nginx.
+ .
+ Please select the web server that you would like to be
+ configured automatically for Vidjil.
+
+Template: vidjil/restart-webserver
+Type: boolean
+Default: true
+Description: Should the webserver(s) be restarted now?
+ The changes will only be applied after restarting the configured webserver(s).
+
+Template: vidjil/configure-startup
+Type: select
+Choices: init, systemd
+Description: Startup system to configure automatically
+ Vidjil Server uses a number of startup scripts to ensure the system will not be
+ down after rebooting.
+ .
+ Please select which one you would like to use.
+
+Template: vidjil/start-services
+Type: boolean
+Default: false
+Description: Should the services be started now ?
+ In some cases it may not be desirable to start services straight away.
+
+Template: vidjil/web2py-password
+Type: string
+Default: VidjilAdm1n
+Description: Web2py admin password:
+ Please enter the chosen password to access the Web2py admin interface.
+
+Template: vidjil/ssl-country
+Type: string
+Default: AU
+Description: Country Code (2 letter code)
+ We will now proceed to generate a temporary self-signed certificate.
+ Please enter your country code.
+
+Template: vidjil/ssl-state
+Type: string
+Default: Some-State
+Description: State or Province (full name)
+ Please enter your state or province.
+
+Template: vidjil/ssl-locality
+Type: string
+Description: Locality Name (eg city)
+ Please enter your locality name.
+
+Template: vidjil/ssl-organisation
+Type: string
+Default: Internet Widgits Pty Ltd
+Description: Organisation name (eg company)
+ Please enter the name of your organisation.
+
+Template: vidjil/ssl-section
+Type: string
+Description: Unit Name (eg section)
+ Please enter the name of your unit.
+
+Template: vidjil/ssl-name
+Type: string
+Description: Common Name (eg server FQDN or Your name)
+ Please enter a common name.
+
+Template: vidjil/ssl-email
+Type: string
+Description: Email Address

packaging/server/debian/vidjil-server-docs.docs

+README.source
+README.Debian
+README

packaging/server/debian/vidjil-server.config

+#!/bin/sh
+
+set -e
+
+. /usr/share/debconf/confmodule
+
+db_input high vidjil/reconfigure-webserver || true
+db_go || true
+db_input high vidjil/restart-webserver || true
+db_go || true
+db_input high vidjil/web2py-password || true
+db_go || true
+db_input high vidjil/configure-startup || true
+db_go || true
+db_input medium vidjil/start-services || true
+db_go || true
+db_input high vidjil/ssl-country || true
+db_go || true
+db_input high vidjil/ssl-state || true
+db_go || true
+db_input high vidjil/ssl-locality || true
+db_go || true
+db_input high vidjil/ssl-organisation || true
+db_go || true
+db_input high vidjil/ssl-section || true
+db_go || true
+db_input high vidjil/ssl-name || true
+db_go || true
+db_input high vidjil/ssl-email || true
+db_go || true

packaging/server/debian/vidjil-server.postrm

+#!/bin/bash
+set -e
+
+# summary of how this script can be called:
+#        * <postrm> `remove'
+#        * <postrm> `purge'
+#        * <old-postrm> `upgrade' <new-version>
+#        * <new-postrm> `failed-upgrade' <old-version>
+#        * <new-postrm> `abort-install'
+#        * <new-postrm> `abort-install' <old-version>
+#        * <new-postrm> `abort-upgrade' <old-version>
+#        * <disappearer's-postrm> `disappear' <r>overwrit>r> <new-version>
+# for details, see http://www.debian.org/doc/debian-policy/ or
+# the debian-policy package
+
+. /usr/share/debconf/confmodule
+
+# Source dbconfig-common functions
+if [ -f /usr/share/dbconfig-common/dpkg/postrm.mysql   ]; then
+    . /usr/share/dbconfig-common/dpkg/postrm.mysql
+fi
+
+purge_nginx() {
+    nginx_conf='/etc/nginx/conf.d/web2py'
+    if [ -d "$nginx_conf"  ]; then