Merge branch 'feature-s/packageless_docker' into 'dev'

merge -- packageless docker, server functional tests

See merge request !225

.gitlab-ci.yml

@@ -23,6 +23,7 @@ stages:
   - valgrind_functional
   - multiple_tests
   - prepare_deploy
+  - deploy_docker
   - deploy_prod
 
 # Anchors
@@ -267,6 +268,28 @@ test_server_unit:
     tags:
       - web
 
+test_server_functional:
+  stage: test_functional
+  script:
+    - docker build --no-cache --build-arg git_branch=$CI_COMMIT_REF_NAME --build-arg build_env=TEST -t "vidjil/server:test" docker/vidjil-server
+    - docker build --no-cache --build-arg git_branch=$CI_COMMIT_REF_NAME --build-arg build_env=TEST -t "vidjil/client:test" docker/vidjil-client
+    - cd docker && docker-compose up -d && cd ..
+    - sed -i "s/^python\ \.\.\/\.\.\/\.\./docker\ exec\ docker_uwsgi_1\ python\ \/usr\/share\/vidjil\/server\/web2py/" server/web2py/applications/vidjil/tests/init_func_test_db.sh
+    - docker exec docker_uwsgi_1 sed -i "s/^\(FILE_SOURCE .*\)/FILE_SOURCE = '\/usr\/share\/vidjil\/demo'/" /usr/share/vidjil/server/web2py/applications/vidjil/modules/defs.py
+    - docker exec docker_nginx_1 make -C /usr/share/vidjil browser
+    - source /etc/profile.d/rvm.sh
+    - rvm use 2.1.1
+    - HEADLESS=1 make functional_server || (cd docker && docker-compose stop; false)
+    - cd docker && docker-compose stop
+  only:
+    - /^feature-.*s.*\/.*$/
+    - /^hotfix-.*s.*\/.*$/
+    - prod-server
+    - dev
+  tags:
+    - x86_64
+    - docker
+
 # Deployment
 
 deploy_review:
@@ -285,6 +308,18 @@ deploy_review:
   tags:
     - web
 
+deploy_docker:
+  stage: deploy_docker
+  script:
+    - docker tag $CI_BUILD_REF_SLUG vidjil/vidjil:latest
+    - docker push vidjil/vidjil:latest
+  when: manual
+  only:
+    - prod-server
+  tags:
+    - web
+    - docker
+
 deploy_prod:
   stage: deploy_prod
   script:

Makefile

@@ -39,6 +39,12 @@ headless_browser:
 unit_server:
 	$(MAKE) -C server/ unit
 
+functional_server:
+	$(MAKE) -C server functional
+
+headless_server:
+	$(MAKE) -C server headless
+
 ###
 
 data:

browser/js/form_builder.js

@@ -364,6 +364,7 @@ FileFormBuilder.prototype.build_jstree = function() {
     var self = this;
     var w = this.build_wrapper();
     var d = document.createElement('div');
+    d.id = "jstree_field_" + self.index;
     w.appendChild(d);
 
     d.className += " jstree_field form-control";

browser/test/Makefile

@@ -8,10 +8,10 @@ headless:
 	HEADLESS=1 $(MAKE) external
 
 functional: ../../doc/analysis-example1.vidjil
-	sh launch_functional_tests "functional/test_*rb"
+	sh launch_functional_tests functional/test_*rb
 
 external: ../../doc/analysis-example1.vidjil
-	sh launch_functional_tests "functional/external_test_*rb"
+	sh launch_functional_tests functional/external_test_*rb
 
 functional-rbx: ../../doc/analysis-example1.vidjil
 	LIVE_SERVER=http://app.vidjil.org/browser/ ruby test_browser.rb

browser/test/launch_functional_tests

-# sh launch_functional_tests "functional/test_*rb"
+# sh launch_functional_tests functional/test_*rb
 
-FUNCTIONAL_TESTS=$1
+FUNCTIONAL_TESTS=$*
 
 rename_reports_in() {
     dir="$1"

doc/dev.org

@@ -646,7 +646,6 @@ make functional
  stop individual services.
  The services are as follows:
    - mysql        The database
-   - postfix      The mailing server
    - uwsgi        The Web2py backend server
    - fuse         The XmlRPCServer that handles custom fuses (for comparing
      samples)
@@ -665,6 +664,10 @@ make functional
    vidjil image you want to use. Usually this will be vidjil/vidjil:latest,
    but more tags are available at https://hub.docker.com/r/vidjil/vidjil/tags/.
 
+   You may also want to uncomment the volume in the fuse volume block "-
+   ./vidjil/conf:/etc/vidjil" this will provide easier access to all of the
+   configuration files, allowing for tweaks.
+
    Running the following command will automatically download any missing
    images and start the environment:
 
@@ -683,34 +686,45 @@ make functional
 
 ** Building images for DockerHub
    Make sure your Dockerfile is up to date with any changes you may want to
-   make to the containers. And make sure the latest vidjil packages have been
-   deployed to the vidjil repository since they will be used as the source
-   for the actual vidjil application.
+   make to the containers. The Dockerfile accepts some build arguments:
+   - build-env: TEST or PRODUCTION. If unspecified, PRODUCTION is assumed.
+     The main difference is that TEST will build the image with an HTTP
+     configuration whereas PRODUCTION uses HTTPS.
+   - git_repo : The repository to build the image from. By default, our main
+     repository is assumed.
+   - git_branch : The git branch to clone from the repository. By default:
+     dev.
 
    #+BEGIN_SRC sh
-     docker-compose build
+     docker build --build-arg build_env=PRODUCTION --build-arg git_branch=<my_feature_branch> docker/vidjil-client -t vidjil/client:<version>
+     docker build --build-arg build_env=PRODUCTION --build-arg git_branch=<my_feature_branch> docker/vidjil-server -t vidjil/server:<version>
    #+END_SRC
 
    Tag the image you have just built:
 
    #+BEGIN_SRC sh
-     docker tag vidjil:test vidjil/vidjil:<version>
-   #+END_SRC
-     or
-   #+BEGIN_SRC sh
-     docker tag vidjil:test vidjil/vidjil:<latestversion>
+     docker tag vidjil:test vidjil/client:latest
+     docker tag vidjil:test vidjil/server:latest
    #+END_SRC
 
    Push the image to DockerHub:
 
    #+BEGIN_SRC sh
-     docker push vidjil/vidjil:<tag>
+     docker push vidjil/client:<tag>
+     docker push vidjil/server:<tag>
    #+END_SRC
 
    You may be required to log in, in which case you can consult
    https://docs.docker.com/engine/reference/commandline/login/ for more
    information.
 
+   If you encounter an issue where docker is unable to access
+   archive.ubuntu.org then you may need to add your dns to /etc/docker/daemon.json
+   #+BEGIN_SRC json
+      {
+          "dns":["dns1", "dns2"]
+      }
+
 * Migrating Data
 ** Database
    The easiest way to perform a database migration is to first extract the

docker/docker-compose.yml

@@ -7,29 +7,19 @@ services:
         volumes:
             - ./mysql/:/docker-entrypoint-initdb.d/
             - /opt/vidjil/mysql:/var/lib/mysql
-    postfix:
-        image: catatnight/postfix
-        environment:
-            - maildomain=mail.example.com
-            - smtp_user=user:pwd
-        ports:
-                - "25:25"
     uwsgi:
-        build: ./vidjil
-        image: vidjil:test
+        image: vidjil/server:test
         command: bash /entrypoints/uwsgi-entrypoint.sh
         volumes_from:
             - fuse
         depends_on:
             - mysql
-            - postfix
             - fuse
         links:
             - mysql:mysql
-            - postfix:postfix
             - fuse:fuse
     fuse:
-        image: vidjil:test
+        image: vidjil/server:test
         command: bash /entrypoints/fuse-entrypoint.sh
         volumes:
             - /opt/vidjil/databases:/usr/share/vidjil/server/web2py/applications/vidjil/databases
@@ -37,12 +27,11 @@ services:
             - /opt/vidjil/result/tmp:/mnt/result/tmp
             - /opt/vidjil/upload/uploads:/mnt/upload/uploads
             - /opt/vidjil/log:/var/vidjil
-            - /opt/vidjil/log/nginx:/var/log/nginx
             - /opt/vidjil/log/uwsgi:/var/log/uwsgi
             - /opt/vidjil/backup:/mnt/backup
-            - ./vidjil/conf:/etc/vidjil
+            #- ./vidjil-server/conf:/etc/vidjil
     nginx:
-        image: vidjil:test
+        image: vidjil/client:test
         depends_on:
             - uwsgi
         links:
@@ -50,13 +39,14 @@ services:
         ports:
             - "80:80"
             - "443:443"
-        #command: bash /entrypoints/nginx-entrypoint.sh
-        command: bash -c "spawn-fcgi -u www-data -s /var/run/fcgiwrap.socket /usr/sbin/fcgiwrap ; nginx -g 'daemon off;'"
-        volumes_from:
-            - uwsgi
+        command: bash /entrypoints/nginx-entrypoint.sh
+        volumes:
+            - /opt/vidjil/log/nginx:/var/log/nginx
+            - /etc/nginx/ssl:/etc/nginx/ssl
+            - ../germline:/usr/share/vidjil/germline
 
     workers:
-        image: vidjil:test
+        image: vidjil/server:test
         command: /usr/local/bin/gosu www-data python /usr/share/vidjil/server/web2py/web2py.py -K vidjil,vidjil,vidjil
         volumes_from:
             - uwsgi
@@ -66,7 +56,7 @@ services:
         links:
             - mysql
     backup:
-        image: vidjil:test
+        image: vidjil/server:test
         command: bash -c "echo '0 5 * * * cd /usr/share/vidjil/server; sh backup.sh /mnt/backup 2>1&' > /etc/cron.d/backup-cron && cron"
         volumes_from:
             - uwsgi
@@ -82,5 +72,4 @@ services:
         depends_on:
             - uwsgi
         links:
-            - postfix:postfix
             - uwsgi:uwsgi

docker/reporter/Dockerfile

-from vidjil:test
+from vidjil/server:test
 
 run apt-get clean && rm -rf /var/lib/apt/lists/* && apt-get update && apt-get install -y cron python python-pip iputils-ping
 

docker/vidjil-client/Dockerfile

+from nginx:1.15.0
+
+label version="1.1"
+label description="An NGINX based docker image which comes \
+with a full installation of the Vidjil client."
+
+run sed -i 's/^\(user .*\)$/user www-data;/' /etc/nginx/nginx.conf
+
+run set -x \
+	&& apt-get update && apt-get install -y --no-install-recommends wget make unzip git sudo curl fcgiwrap && rm -rf /var/lib/apt/lists/*
+
+arg git_branch=dev
+arg remote_repo=https://gitlab.inria.fr/vidjil/vidjil.git
+run cd /usr/share/ && git config --global http.sslVerify false && git clone -b $git_branch $remote_repo
+
+copy ./conf/nginx_gzip_static.conf /etc/nginx/conf.d/web2py/gzip_static.conf
+copy ./conf/nginx_gzip.conf /etc/nginx/conf.d/web2py/gzip.conf
+copy ./conf/uwsgi.conf /etc/nginx/conf.d/web2py/uwsgi.conf
+add ./scripts/install.sh /opt/install_scripts/install.sh
+copy ./conf/conf.js /opt/vidjil_conf/conf.js
+copy ./conf/conf_http.js /opt/vidjil_conf/conf_http.js
+copy ./conf/nginx_web2py /opt/vidjil_conf/web2py
+copy ./conf/nginx_web2py_http /opt/vidjil_conf/web2py_http
+copy ./conf/Gemfile /usr/share/vidjil/Gemfile
+copy ./conf/align.cgi /usr/share/vidjil/browser/cgi/align.cgi
+copy ./conf/similarity.cgi /usr/share/vidjil/browser/cgi/similarity.cgi
+
+run cd /usr/share/vidjil/browser/css/icons && make
+
+arg build_env='PRODUCTION'
+env BUILD_ENV $build_env
+
+run mkdir /etc/vidjil
+run rm /etc/nginx/conf.d/default.conf
+run chmod +x /opt/install_scripts/install.sh; sync && /opt/install_scripts/install.sh
+run ln -s /etc/vidjil/conf.js /usr/share/vidjil/browser/js/conf.js
+
+copy ./scripts/nginx-entrypoint.sh /entrypoints/nginx-entrypoint.sh
+run chown -R www-data:www-data /usr/share/vidjil
+run useradd -ms /bin/bash vidjil && usermod -aG sudo vidjil

docker/vidjil-client/conf/Gemfile

+source 'http://rubygems.org'
+gem 'minitest'
+gem 'minitest-ci'
+gem 'watir'
+gem 'watir-webdriver'
+gem 'test-unit'
+gem 'headless'

docker/vidjil/conf/conf.js → docker/vidjil-client/conf/conf.js

@@ -8,7 +8,7 @@ var config = {
     /* Used for the 'align' script 
      * If this is not defined, the 'align' button is not available
      */    
-    "cgi_address" : "http://localhost/cgi/", // Public test server
+    "cgi_address" : "https://localhost/cgi/", // Public test server
     // "cgi_address" : "http://127.0.1.1/cgi-bin/",
 
     /* The following options control how the user may have access to .vidjil files.

docker/vidjil-client/conf/conf_http.js

+/* 
+ * Vidjil browser, main configuration file
+ * This file must be named 'js/conf.js' to be taken into account
+ * */
+
+var config = {
+
+    /* Used for the 'align' script 
+     * If this is not defined, the 'align' button is not available
+     */    
+    "cgi_address" : "http://localhost/cgi/", // Public test server
+    // "cgi_address" : "http://127.0.1.1/cgi-bin/",
+
+    /* The following options control how the user may have access to .vidjil files.
+     * Any combination of 1), 2) and 3) should work
+     */
+
+    /* 1) Patient database */
+    "use_database" : true,
+    "db_address" : "http://localhost/vidjil/", // Public test server
+    "login" : "",
+    "password" : "",
+
+    /* 2) and 3) Static files
+    /* - relative paths if Vidjil browser is online on the same server
+     * - absolute paths to a CORS active server only if browser is offline or on another server 
+     */
+
+    /* 2) Menu with a list of static files */
+    "file_menu" : {
+        "path" : "/browser/data/",
+        "file" : [
+            "Stanford-S22.vidjil",
+            "L2-LIL-2.vidjil"
+        ]
+    },
+    
+    /* 3) Static file autoload, possibly with an .analysis file */
+    // "autoload" : "data/Stanford-S22.vidjil",
+    // "autoload_analysis" : "data/Stanford-S22.analysis"
+
+    // Proxy config for IMGT querying
+    "proxy": "http://localhost/vidjil/proxy/imgt"
+}

docker/vidjil/conf/nginx_web2py → docker/vidjil-client/conf/nginx_web2py

@@ -36,7 +36,7 @@ server {
         ## from the previous server instance to manage static files
 
     location /browser {
-        root /var/www;
+        root /usr/share/vidjil;
         expires 1h;
         add_header Cache-Control must-revalidate;
 

docker/vidjil-client/conf/nginx_web2py_http

+upstream uwsgi {
+    server uwsgi:8000;
+}
+
+server {
+        listen 80 default_server;
+        keepalive_timeout    70;
+
+        uwsgi_max_temp_file_size 20480m;
+
+        location / {
+            include /etc/nginx/conf.d/web2py/uwsgi.conf;
+            proxy_read_timeout 600;
+            client_max_body_size 20G;
+            ###
+        
+        }
+        ## if you serve static files through https, copy here the section
+        ## from the previous server instance to manage static files
+
+    location /browser {
+        root /usr/share/vidjil;
+        expires 1h;
+        add_header Cache-Control must-revalidate;
+
+        error_page 405 = $uri;
+    
+    }
+
+    location /germline {
+        root /usr/share/vidjil/;
+        expires 1h;
+        add_header Cache-Control must-revalidate;
+
+        error_page 405 = $uri;
+    
+    }
+
+    ###to enable correct use of response.static_version
+    #location ~* ^/(\w+)/static(?:/_[\d]+\.[\d]+\.[\d]+)?/(.*)$ {
+        #    alias /usr/share/vidjil/server/web2py/applications/$1/static/$2;
+        #    expires max;
+        #
+    #}
+    ###
+
+    client_max_body_size 20G;
+
+    location /cgi/ {
+        gzip off;
+        root  /usr/share/vidjil/browser/;
+        # Fastcgi socket
+        fastcgi_pass  unix:/var/run/fcgiwrap.socket;
+        # Fastcgi parameters, include the standard ones
+        include /etc/nginx/fastcgi_params;
+        # Adjust non standard parameters (SCRIPT_FILENAME)
+        fastcgi_param SCRIPT_FILENAME  $document_root$fastcgi_script_name;
+    
+    }
+
+    location /vidjil/file/upload {
+        include /etc/nginx/conf.d/web2py/uwsgi.conf;
+        uwsgi_read_timeout 10m;
+        client_max_body_size 20G;
+    }
+}

docker/vidjil-client/conf/uwsgi_upstream.conf

+

docker/vidjil-client/scripts/install.sh

+#!/bin/bash
+echo "${BUILD_ENV}"
+if [ "${BUILD_ENV}" = "TEST"  ]; then
+    cp -avr /opt/vidjil_conf/conf_http.js /etc/vidjil/conf.js
+    cp -avr /opt/vidjil_conf/web2py_http /etc/nginx/conf.d/web2py.conf
+else
+    cp -avr /opt/vidjil_conf/conf.js /etc/vidjil/conf.js
+    cp -avr /opt/vidjil_conf/web2py /etc/nginx/conf.d/web2py.conf
+fi;
+

docker/vidjil-client/scripts/nginx-entrypoint.sh

+#!/bin/bash
+spawn-fcgi -u www-data -s /var/run/fcgiwrap.socket /usr/sbin/fcgiwrap
+nginx -g 'daemon off;'

docker/vidjil/Dockerfile → docker/vidjil-server/Dockerfile

 from ubuntu:16.04
 
-label version="1.0"
+label version="1.1"
 label description="An Ubuntu based docker image which comes \
 with a full installation of the Vidjil algoright and browser/server."
 
 env GOSU_VERSION 1.7
 run set -x \
-	&& apt-get update && apt-get install -y --no-install-recommends ca-certificates wget cron && rm -rf /var/lib/apt/lists/* \
+	&& apt-get update && apt-get install -y --no-install-recommends ca-certificates wget cron unzip make python ipython python-enum34 python-requests git && rm -rf /var/lib/apt/lists/* \
 	&& wget -O /usr/local/bin/gosu "https://github.com/tianon/gosu/releases/download/$GOSU_VERSION/gosu-$(dpkg --print-architecture)" \
 	&& wget -O /usr/local/bin/gosu.asc "https://github.com/tianon/gosu/releases/download/$GOSU_VERSION/gosu-$(dpkg --print-architecture).asc" \
 	&& export GNUPGHOME="$(mktemp -d)" \
 	&& gpg --keyserver ha.pool.sks-keyservers.net --recv-keys B42F6819007F00F88E364FD4036A9C25BF357DD4 \
 	&& gpg --batch --verify /usr/local/bin/gosu.asc /usr/local/bin/gosu \
 	&& rm -r "$GNUPGHOME" /usr/local/bin/gosu.asc \
-	&& chmod +x /usr/local/bin/gosu \
-	&& apt-get purge -y --auto-remove ca-certificates wget
-
-run echo deb http://vda.vidjil.org:8080/archive sid/all/ >> /etc/apt/sources.list
-run echo deb http://vda.vidjil.org:8080/archive sid/amd64/ >> /etc/apt/sources.list
-run echo deb http://vda.vidjil.org:8080/archive wheezy/all/ >> /etc/apt/sources.list
-run echo deb http://vda.vidjil.org:8080/archive wheezy/amd64/ >> /etc/apt/sources.list
-
-run echo "vidjil-server vidjil/reconfigure-webserver string nginx" > preseed.txt
-run echo "vidjil-server vidjil/restart-webserver boolean false" >> preseed.txt
-run echo "vidjil-server vidjil/configure-startup string none" >> preseed.txt
-run echo "vidjil-server vidjil/start-services boolean false" >> preseed.txt
-run echo "vidjil-server vidjil/web2py-password string VidjilAdm1n" >> preseed.txt
-run echo "vidjil-server vidjil/ssl-country string AU" >> preseed.txt
-run echo "vidjil-server vidjil/ssl-state string Some-State" >> preseed.txt
-run echo "vidjil-server vidjil/ssl-locality string Some-Locality" >> preseed.txt
-run echo "vidjil-server vidjil/ssl-organisation string Internet Widgits Pty Ltd" >> preseed.txt
-run echo "vidjil-server vidjil/ssl-section string Some-Section" >> preseed.txt
-run echo "vidjil-server vidjil/ssl-name string VidjilTeam" >> preseed.txt
-run echo "vidjil-server vidjil/ssl-email string team@vidjil.org" >> preseed.txt
-
-run debconf-set-selections preseed.txt
-run gpg --recv-keys --keyserver vda.vidjil.org 8EA70E25
-run apt-get update ; apt-get install --no-install-recommends --no-install-suggests -y -q sudo curl apt-utils uwsgi-plugin-python nginx-full fcgiwrap
-run apt-get update ; DEBIAN_FRONTEND=noninteractive apt-get -y -q --allow-unauthenticated install vidjil-archive-keyring
-run apt-get update ; DEBIAN_FRONTEND=noninteractive apt-get install --no-install-recommends --no-install-suggests -y -q --allow-unauthenticated vidjil-server
-
-run cd /usr/share/vidjil/germline && make
-
-#run ln -sf /dev/stdout /var/log/nginx/access.log \
-#	&& ln -sf /dev/stderr /var/log/nginx/error.log
-
-copy ./conf/nginx_gzip_static.conf /etc/nginx/conf.d/web2py/gzip_static.conf
-copy ./conf/nginx_gzip.conf /etc/nginx/conf.d/web2py/gzip.conf
-copy ./conf/uwsgi.conf /etc/nginx/conf.d/web2py/uwsgi.conf
-copy ./conf/nginx_web2py /etc/nginx/sites-enabled/web2py
-copy ./conf/nginx_web2py /etc/nginx/sites-available/web2py
+	&& chmod +x /usr/local/bin/gosu
+
+run apt-get update ; apt-get install --no-install-recommends --no-install-suggests -y -q sudo curl apt-utils uwsgi-plugin-python
+
+arg git_branch=dev
+arg remote_repo=https://gitlab.inria.fr/vidjil/vidjil.git
+run cd /usr/share/ && git config --global http.sslVerify false && git clone -b $git_branch $remote_repo
+run cd /usr/share/vidjil/server && wget --no-check-certificate https://mdipierro.pythonanywhere.com/examples/static/web2py_src.zip && unzip web2py_src.zip && unlink web2py_src.zip
+
+add ./scripts/install.sh /opt/install_scripts/install.sh
+copy ./conf/defs.py /opt/vidjil_conf/defs.py
+copy ./conf/defs_http.py /opt/vidjil_conf/defs_http.py
+copy ./conf/base.py /usr/share/vidjil/server/web2py/gluon/packages/dal/pydal/base.py
+
+arg build_env='PRODUCTION'
+env BUILD_ENV $build_env
 
 copy ./conf/uwsgi.ini /etc/uwsgi/apps-enabled/web2py.ini
-copy ./conf/defs.py /etc/vidjil/defs.py
-copy ./conf/conf.js /etc/vidjil/conf.js
-copy ./scripts/nginx-entrypoint.sh /entrypoints/nginx-entrypoint.sh
+run mkdir /etc/vidjil
+copy ./conf/wsgihandler.py /etc/vidjil/wsgihandler.py
+run chmod +x /opt/install_scripts/install.sh; sync && /opt/install_scripts/install.sh
+run ln -s /etc/vidjil/defs.py /usr/share/vidjil/server/web2py/applications/vidjil/modules/defs.py
+run ln -s /etc/vidjil/wsgihandler.py /usr/share/vidjil/server/web2py/wsgihandler.py
+
+run wget http://www.vidjil.org/releases/vidjil-latest_x86_64 && mv vidjil-latest_x86_64 /usr/share/vidjil/vidjil-algo && cd /usr/share/vidjil/ && chmod +x vidjil-algo && cd germline && make
+
 copy ./scripts/uwsgi-entrypoint.sh /entrypoints/uwsgi-entrypoint.sh
 copy ./scripts/fuse-entrypoint.sh /entrypoints/fuse-entrypoint.sh
 run mkdir /usr/share/vidjil/server/web2py/applications/vidjil/databases && touch /usr/share/vidjil/server/web2py/applications/vidjil/databases/sql.log
+run mkdir /var/vidjil
 run touch /var/vidjil/vidjil.log && touch /var/vidjil/vidjil-debug.log
-run adduser vidjil && usermod -aG sudo vidjil
+run cd /usr/share/vidjil/server/web2py && python -c "from gluon.main import save_password; save_password('default',443)"
+run chown -R www-data:www-data /usr/share/vidjil
+run useradd -ms /bin/bash vidjil && usermod -aG sudo vidjil

docker/vidjil/conf/defs.py → docker/vidjil-server/conf/defs.py

@@ -37,7 +37,7 @@ BASENAME_OUT_VIDJIL_ID = '%06d'
 
 ### Directory for program used in task.py
 ### relative path start from server/web2py
-DIR_VIDJIL = '/usr/bin/'
+DIR_VIDJIL = '/usr/share/vidjil/'
 DIR_FUSE = '../../tools'
 DIR_MIXCR = '/usr/bin'
 DIR_IGREC = '/usr/local/bin/'
@@ -70,8 +70,10 @@ SCHEDULER_HEARTBEAT = 5
 
 # Directory to search for files
 FILE_SOURCE = '/mnt/data/src'
-FILE_TYPES = ['fasta', 'fastq', 'fastq.gz']
+FILE_TYPES = ['fasta', 'fastq', 'fastq.gz', 'fa']
 
 SET_TYPE_PATIENT = 'patient'
 SET_TYPE_RUN= 'run'
 SET_TYPE_GENERIC = 'generic'
+
+REQUIRE_HTTPS = True

docker/vidjil-server/conf/defs_http.py

+###
+### Vidjil server, main configuration file
+### This file must be named 'modules/defs.py' to be taken into account
+###
+
+### Email notifications for server errors
+SMTP_SERVER = 'localhost'
+# SMTP_SERVER = 'logging' # no mail
+FROM_EMAIL = 'notifications@vidjil.org'
+ADMIN_EMAILS = ['notifications@vidjil.org']
+
+### address for the sql database
+###
+DB_ADDRESS = 'sqlite://storage.sqlite'
+DB_ADDRESS = 'mysql://vidjil:rootpass@mysql/vidjil'
+DB_POOL_SIZE = 4
+
+### Backup file
+DB_BACKUP_FILE = 'backup.csv'
+
+### Upload directory for .fasta/.fastq.
+### Old sequences files could be thrown away.
+### No regular backup.
+
+DIR_SEQUENCES = '/mnt/upload/uploads/'
+
+### Upload directory for .vidjil/.fused/.analysis
+### Regularly backuped
+
+DIR_RESULTS = '/mnt/result/results/'
+
+### Temporary directory to store vidjil results, and basename of files in this directory
+### Formatted with 'data_id' in models/task.py
+DIR_OUT_VIDJIL_ID = '/mnt/result/tmp/out-%06d/'
+DIR_PRE_VIDJIL_ID = '/mnt/result/tmp/pre/out-%06d/'
+BASENAME_OUT_VIDJIL_ID = '%06d'
+
+### Directory for program used in task.py
+### relative path start from server/web2py
+DIR_VIDJIL = '/usr/share/vidjil/'
+DIR_FUSE = '../../tools'
+DIR_MIXCR = '/usr/bin'
+DIR_IGREC = '/usr/local/bin/'
+DIR_GERMLINE = '/usr/share/vidjil/germline'
+### Port on which to run the fuse server
+### Used in models/task.py and in /server/fuse_server.py
+FUSE_SERVER = 'fuse'
+PORT_FUSE_SERVER = 12789
+
+### Log files
+DIR_LOG = '/var/vidjil/'
+LOG_INFO = DIR_LOG + 'vidjil.log'
+LOG_DEBUG = DIR_LOG + 'vidjil-debug.log'
+
+### Timeouts
+# Delay in seconds before a task is considered in timeout
+TASK_TIMEOUT = 2 * 60 * 60
+
+### Reverse IP file
+REVERSE_IP = '/home/vidjil/ips.txt'
+
+### Locus (should be parsed from germlines.data)
+LOCUS = ['TRA', 'TRA+D', 'TRB', 'TRG', 'TRD', 'TRD+',
+         'IGH', 'IGH+', 'IGK', 'IGK+', 'IGL']
+
+# Preventu upload and run when 1% space is left in target disk
+FS_LOCK_THRESHHOLD = 1
+
+SCHEDULER_HEARTBEAT = 5
+
+# Directory to search for files
+FILE_SOURCE = '/mnt/data/src'
+FILE_TYPES = ['fasta', 'fastq', 'fastq.gz', 'fa']
+
+SET_TYPE_PATIENT = 'patient'
+SET_TYPE_RUN= 'run'
+SET_TYPE_GENERIC = 'generic'
+
+REQUIRE_HTTPS = False