Merge branch 'feature-s/packageless_docker' into 'dev'

merge -- packageless docker, server functional tests

See merge request !225

.gitlab-ci.yml

@@ -23,6 +23,7 @@ stages:
   - valgrind_functional
   - multiple_tests
   - prepare_deploy
+  - deploy_docker
   - deploy_prod
 
 # Anchors
@@ -267,6 +268,28 @@ test_server_unit:
     tags:
       - web
 
+test_server_functional:
+  stage: test_functional
+  script:
+    - docker build --no-cache --build-arg git_branch=$CI_COMMIT_REF_NAME --build-arg build_env=TEST -t "vidjil/server:test" docker/vidjil-server
+    - docker build --no-cache --build-arg git_branch=$CI_COMMIT_REF_NAME --build-arg build_env=TEST -t "vidjil/client:test" docker/vidjil-client
+    - cd docker && docker-compose up -d && cd ..
+    - sed -i "s/^python\ \.\.\/\.\.\/\.\./docker\ exec\ docker_uwsgi_1\ python\ \/usr\/share\/vidjil\/server\/web2py/" server/web2py/applications/vidjil/tests/init_func_test_db.sh
+    - docker exec docker_uwsgi_1 sed -i "s/^\(FILE_SOURCE .*\)/FILE_SOURCE = '\/usr\/share\/vidjil\/demo'/" /usr/share/vidjil/server/web2py/applications/vidjil/modules/defs.py
+    - docker exec docker_nginx_1 make -C /usr/share/vidjil browser
+    - source /etc/profile.d/rvm.sh
+    - rvm use 2.1.1
+    - HEADLESS=1 make functional_server || (cd docker && docker-compose stop; false)
+    - cd docker && docker-compose stop
+  only:
+    - /^feature-.*s.*\/.*$/
+    - /^hotfix-.*s.*\/.*$/
+    - prod-server
+    - dev
+  tags:
+    - x86_64
+    - docker
+
 # Deployment
 
 deploy_review:
@@ -285,6 +308,18 @@ deploy_review:
   tags:
     - web
 
+deploy_docker:
+  stage: deploy_docker
+  script:
+    - docker tag $CI_BUILD_REF_SLUG vidjil/vidjil:latest
+    - docker push vidjil/vidjil:latest
+  when: manual
+  only:
+    - prod-server
+  tags:
+    - web
+    - docker
+
 deploy_prod:
   stage: deploy_prod
   script:

Makefile

@@ -39,6 +39,12 @@ headless_browser:
 unit_server:
 	$(MAKE) -C server/ unit
 
+functional_server:
+	$(MAKE) -C server functional
+
+headless_server:
+	$(MAKE) -C server headless
+
 ###
 
 data:

browser/js/form_builder.js

@@ -364,6 +364,7 @@ FileFormBuilder.prototype.build_jstree = function() {
     var self = this;
     var w = this.build_wrapper();
     var d = document.createElement('div');
+    d.id = "jstree_field_" + self.index;
     w.appendChild(d);
 
     d.className += " jstree_field form-control";

browser/test/Makefile

@@ -8,10 +8,10 @@ headless:
 	HEADLESS=1 $(MAKE) external
 
 functional: ../../doc/analysis-example1.vidjil
-	sh launch_functional_tests "functional/test_*rb"
+	sh launch_functional_tests functional/test_*rb
 
 external: ../../doc/analysis-example1.vidjil
-	sh launch_functional_tests "functional/external_test_*rb"
+	sh launch_functional_tests functional/external_test_*rb
 
 functional-rbx: ../../doc/analysis-example1.vidjil
 	LIVE_SERVER=http://app.vidjil.org/browser/ ruby test_browser.rb

browser/test/launch_functional_tests

-# sh launch_functional_tests "functional/test_*rb"
+# sh launch_functional_tests functional/test_*rb
 
-FUNCTIONAL_TESTS=$1
+FUNCTIONAL_TESTS=$*
 
 rename_reports_in() {
     dir="$1"

doc/dev.org

@@ -646,7 +646,6 @@ make functional
  stop individual services.
  The services are as follows:
    - mysql        The database
-   - postfix      The mailing server
    - uwsgi        The Web2py backend server
    - fuse         The XmlRPCServer that handles custom fuses (for comparing
      samples)
@@ -665,6 +664,10 @@ make functional
    vidjil image you want to use. Usually this will be vidjil/vidjil:latest,
    but more tags are available at https://hub.docker.com/r/vidjil/vidjil/tags/.
 
+   You may also want to uncomment the volume in the fuse volume block "-
+   ./vidjil/conf:/etc/vidjil" this will provide easier access to all of the
+   configuration files, allowing for tweaks.
+
    Running the following command will automatically download any missing
    images and start the environment:
 
@@ -683,34 +686,45 @@ make functional
 
 ** Building images for DockerHub
    Make sure your Dockerfile is up to date with any changes you may want to
-   make to the containers. And make sure the latest vidjil packages have been
-   deployed to the vidjil repository since they will be used as the source
-   for the actual vidjil application.
+   make to the containers. The Dockerfile accepts some build arguments:
+   - build-env: TEST or PRODUCTION. If unspecified, PRODUCTION is assumed.
+     The main difference is that TEST will build the image with an HTTP
+     configuration whereas PRODUCTION uses HTTPS.
+   - git_repo : The repository to build the image from. By default, our main
+     repository is assumed.
+   - git_branch : The git branch to clone from the repository. By default:
+     dev.
 
    #+BEGIN_SRC sh
-     docker-compose build
+     docker build --build-arg build_env=PRODUCTION --build-arg git_branch=<my_feature_branch> docker/vidjil-client -t vidjil/client:<version>
+     docker build --build-arg build_env=PRODUCTION --build-arg git_branch=<my_feature_branch> docker/vidjil-server -t vidjil/server:<version>
    #+END_SRC
 
    Tag the image you have just built:
 
    #+BEGIN_SRC sh
-     docker tag vidjil:test vidjil/vidjil:<version>
-   #+END_SRC
-     or
-   #+BEGIN_SRC sh
-     docker tag vidjil:test vidjil/vidjil:<latestversion>
+     docker tag vidjil:test vidjil/client:latest
+     docker tag vidjil:test vidjil/server:latest
    #+END_SRC
 
    Push the image to DockerHub:
 
    #+BEGIN_SRC sh
-     docker push vidjil/vidjil:<tag>
+     docker push vidjil/client:<tag>
+     docker push vidjil/server:<tag>
    #+END_SRC
 
    You may be required to log in, in which case you can consult
    https://docs.docker.com/engine/reference/commandline/login/ for more
    information.
 
+   If you encounter an issue where docker is unable to access
+   archive.ubuntu.org then you may need to add your dns to /etc/docker/daemon.json
+   #+BEGIN_SRC json
+      {
+          "dns":["dns1", "dns2"]
+      }
+
 * Migrating Data
 ** Database
    The easiest way to perform a database migration is to first extract the

docker/docker-compose.yml

@@ -7,29 +7,19 @@ services:
         volumes:
             - ./mysql/:/docker-entrypoint-initdb.d/
             - /opt/vidjil/mysql:/var/lib/mysql
-    postfix:
-        image: catatnight/postfix
-        environment:
-            - maildomain=mail.example.com
-            - smtp_user=user:pwd
-        ports:
-                - "25:25"
     uwsgi:
-        build: ./vidjil
-        image: vidjil:test
+        image: vidjil/server:test
         command: bash /entrypoints/uwsgi-entrypoint.sh
         volumes_from:
             - fuse
         depends_on:
             - mysql
-            - postfix
             - fuse
         links:
             - mysql:mysql
-            - postfix:postfix
             - fuse:fuse
     fuse:
-        image: vidjil:test
+        image: vidjil/server:test
         command: bash /entrypoints/fuse-entrypoint.sh
         volumes:
             - /opt/vidjil/databases:/usr/share/vidjil/server/web2py/applications/vidjil/databases
@@ -37,12 +27,11 @@ services:
             - /opt/vidjil/result/tmp:/mnt/result/tmp
             - /opt/vidjil/upload/uploads:/mnt/upload/uploads
             - /opt/vidjil/log:/var/vidjil
-            - /opt/vidjil/log/nginx:/var/log/nginx
             - /opt/vidjil/log/uwsgi:/var/log/uwsgi
             - /opt/vidjil/backup:/mnt/backup
-            - ./vidjil/conf:/etc/vidjil
+            #- ./vidjil-server/conf:/etc/vidjil
     nginx:
-        image: vidjil:test
+        image: vidjil/client:test
         depends_on:
             - uwsgi
         links:
@@ -50,13 +39,14 @@ services:
         ports:
             - "80:80"
             - "443:443"
-        #command: bash /entrypoints/nginx-entrypoint.sh
-        command: bash -c "spawn-fcgi -u www-data -s /var/run/fcgiwrap.socket /usr/sbin/fcgiwrap ; nginx -g 'daemon off;'"
-        volumes_from:
-            - uwsgi
+        command: bash /entrypoints/nginx-entrypoint.sh
+        volumes:
+            - /opt/vidjil/log/nginx:/var/log/nginx
+            - /etc/nginx/ssl:/etc/nginx/ssl
+            - ../germline:/usr/share/vidjil/germline
 
     workers:
-        image: vidjil:test
+        image: vidjil/server:test
         command: /usr/local/bin/gosu www-data python /usr/share/vidjil/server/web2py/web2py.py -K vidjil,vidjil,vidjil
         volumes_from:
             - uwsgi
@@ -66,7 +56,7 @@ services:
         links:
             - mysql
     backup:
-        image: vidjil:test
+        image: vidjil/server:test
         command: bash -c "echo '0 5 * * * cd /usr/share/vidjil/server; sh backup.sh /mnt/backup 2>1&' > /etc/cron.d/backup-cron && cron"
         volumes_from:
             - uwsgi
@@ -82,5 +72,4 @@ services:
         depends_on:
             - uwsgi
         links:
-            - postfix:postfix
             - uwsgi:uwsgi

docker/reporter/Dockerfile

-from vidjil:test
+from vidjil/server:test
 
 run apt-get clean && rm -rf /var/lib/apt/lists/* && apt-get update && apt-get install -y cron python python-pip iputils-ping
 

docker/vidjil-client/Dockerfile

+from nginx:1.15.0
+
+label version="1.1"
+label description="An NGINX based docker image which comes \
+with a full installation of the Vidjil client."
+
+run sed -i 's/^\(user .*\)$/user www-data;/' /etc/nginx/nginx.conf
+
+run set -x \
+	&& apt-get update && apt-get install -y --no-install-recommends wget make unzip git sudo curl fcgiwrap && rm -rf /var/lib/apt/lists/*
+
+arg git_branch=dev
+arg remote_repo=https://gitlab.inria.fr/vidjil/vidjil.git
+run cd /usr/share/ && git config --global http.sslVerify false && git clone -b $git_branch $remote_repo
+
+copy ./conf/nginx_gzip_static.conf /etc/nginx/conf.d/web2py/gzip_static.conf
+copy ./conf/nginx_gzip.conf /etc/nginx/conf.d/web2py/gzip.conf
+copy ./conf/uwsgi.conf /etc/nginx/conf.d/web2py/uwsgi.conf
+add ./scripts/install.sh /opt/install_scripts/install.sh
+copy ./conf/conf.js /opt/vidjil_conf/conf.js
+copy ./conf/conf_http.js /opt/vidjil_conf/conf_http.js
+copy ./conf/nginx_web2py /opt/vidjil_conf/web2py
+copy ./conf/nginx_web2py_http /opt/vidjil_conf/web2py_http
+copy ./conf/Gemfile /usr/share/vidjil/Gemfile
+copy ./conf/align.cgi /usr/share/vidjil/browser/cgi/align.cgi
+copy ./conf/similarity.cgi /usr/share/vidjil/browser/cgi/similarity.cgi
+
+run cd /usr/share/vidjil/browser/css/icons && make
+
+arg build_env='PRODUCTION'
+env BUILD_ENV $build_env
+
+run mkdir /etc/vidjil
+run rm /etc/nginx/conf.d/default.conf
+run chmod +x /opt/install_scripts/install.sh; sync && /opt/install_scripts/install.sh
+run ln -s /etc/vidjil/conf.js /usr/share/vidjil/browser/js/conf.js
+
+copy ./scripts/nginx-entrypoint.sh /entrypoints/nginx-entrypoint.sh
+run chown -R www-data:www-data /usr/share/vidjil
+run useradd -ms /bin/bash vidjil && usermod -aG sudo vidjil

docker/vidjil-client/conf/Gemfile

+source 'http://rubygems.org'
+gem 'minitest'
+gem 'minitest-ci'
+gem 'watir'
+gem 'watir-webdriver'
+gem 'test-unit'
+gem 'headless'

docker/vidjil/conf/conf.js → docker/vidjil-client/conf/conf.js

@@ -8,7 +8,7 @@ var config = {
     /* Used for the 'align' script 
      * If this is not defined, the 'align' button is not available
      */    
-    "cgi_address" : "http://localhost/cgi/", // Public test server
+    "cgi_address" : "https://localhost/cgi/", // Public test server
     // "cgi_address" : "http://127.0.1.1/cgi-bin/",
 
     /* The following options control how the user may have access to .vidjil files.

docker/vidjil-client/conf/conf_http.js

+/* 
+ * Vidjil browser, main configuration file
+ * This file must be named 'js/conf.js' to be taken into account
+ * */
+
+var config = {
+
+    /* Used for the 'align' script 
+     * If this is not defined, the 'align' button is not available
+     */    
+    "cgi_address" : "http://localhost/cgi/", // Public test server
+    // "cgi_address" : "http://127.0.1.1/cgi-bin/",
+
+    /* The following options control how the user may have access to .vidjil files.
+     * Any combination of 1), 2) and 3) should work
+     */
+
+    /* 1) Patient database */
+    "use_database" : true,
+    "db_address" : "http://localhost/vidjil/", // Public test server
+    "login" : "",
+    "password" : "",
+
+    /* 2) and 3) Static files
+    /* - relative paths if Vidjil browser is online on the same server
+     * - absolute paths to a CORS active server only if browser is offline or on another server 
+     */
+
+    /* 2) Menu with a list of static files */
+    "file_menu" : {
+        "path" : "/browser/data/",
+        "file" : [
+            "Stanford-S22.vidjil",
+            "L2-LIL-2.vidjil"
+        ]
+    },
+    
+    /* 3) Static file autoload, possibly with an .analysis file */
+    // "autoload" : "data/Stanford-S22.vidjil",
+    // "autoload_analysis" : "data/Stanford-S22.analysis"
+
+    // Proxy config for IMGT querying
+    "proxy": "http://localhost/vidjil/proxy/imgt"
+}

docker/vidjil/conf/nginx_web2py → docker/vidjil-client/conf/nginx_web2py

@@ -36,7 +36,7 @@ server {
         ## from the previous server instance to manage static files
 
     location /browser {
-        root /var/www;
+        root /usr/share/vidjil;
         expires 1h;
         add_header Cache-Control must-revalidate;
 

docker/vidjil-client/conf/nginx_web2py_http

+upstream uwsgi {
+    server uwsgi:8000;
+}
+
+server {
+        listen 80 default_server;
+        keepalive_timeout    70;
+
+        uwsgi_max_temp_file_size 20480m;
+
+        location / {
+            include /etc/nginx/conf.d/web2py/uwsgi.conf;
+            proxy_read_timeout 600;
+            client_max_body_size 20G;
+            ###
+        
+        }
+        ## if you serve static files through https, copy here the section
+        ## from the previous server instance to manage static files
+
+    location /browser {
+        root /usr/share/vidjil;
+        expires 1h;
+        add_header Cache-Control must-revalidate;
+
+        error_page 405 = $uri;
+    
+    }
+
+    location /germline {
+        root /usr/share/vidjil/;
+        expires 1h;
+        add_header Cache-Control must-revalidate;
+
+        error_page 405 = $uri;
+    
+    }
+
+    ###to enable correct use of response.static_version
+    #location ~* ^/(\w+)/static(?:/_[\d]+\.[\d]+\.[\d]+)?/(.*)$ {
+        #    alias /usr/share/vidjil/server/web2py/applications/$1/static/$2;
+        #    expires max;
+        #
+    #}
+    ###
+
+    client_max_body_size 20G;
+
+    location /cgi/ {
+        gzip off;
+        root  /usr/share/vidjil/browser/;
+        # Fastcgi socket
+        fastcgi_pass  unix:/var/run/fcgiwrap.socket;
+        # Fastcgi parameters, include the standard ones
+        include /etc/nginx/fastcgi_params;
+        # Adjust non standard parameters (SCRIPT_FILENAME)
+        fastcgi_param SCRIPT_FILENAME  $document_root$fastcgi_script_name;
+    
+    }
+
+    location /vidjil/file/upload {
+        include /etc/nginx/conf.d/web2py/uwsgi.conf;
+        uwsgi_read_timeout 10m;
+        client_max_body_size 20G;
+    }
+}

docker/vidjil-client/conf/uwsgi_upstream.conf

+