Merge branch 'feature-s/server_reviews' into 'dev'

Feature s/server reviews

Closes #4167

See merge request !632

.gitlab-ci.yml

@@ -325,7 +325,7 @@ test_server_unit:
   script:
     - docker build --no-cache --build-arg git_branch=$CI_COMMIT_REF_NAME --build-arg build_env=TEST -t "vidjil/server:test" docker/vidjil-server
     - docker build --no-cache --build-arg git_branch=$CI_COMMIT_REF_NAME --build-arg build_env=TEST -t "vidjil/client:test" docker/vidjil-client
-    - sed -i '/\/etc\/nginx\/ssl\:\/etc\/nginx\/ssl/d' ./docker/docker-compose.yml
+    - sed -i '/\:\/etc\/nginx\/ssl/d' ./docker/docker-compose.yml
     - sed -i '/\/var\/lib\/mysql/d' ./docker/docker-compose.yml
     - sed -i 's/\/opt\/vidjil\/databases/databases/' ./docker/docker-compose.yml
     - echo -e "volumes:\n    databases:" >> ./docker/docker-compose.yml
@@ -398,6 +398,32 @@ deploy_review:
   tags:
     - web
 
+deploy_review_server:
+  stage: deploy_review
+  script:
+    - make -C browser icons
+    - cp ./docker/vidjil-client/conf/conf.js ./browser/js/conf.js
+    - sed -i -r "s/localhost/${CI_BUILD_REF_SLUG}.server.ci.vidjil.org/g" ./browser/js/conf.js
+    - sed -r 's/^.*FILE_SOURCE.*$/FILE_SOURCE="\/usr\/share\/vidjil\/demo"/' ./docker/vidjil-server/conf/defs.py > ./server/web2py/applications/vidjil/modules/defs.py
+    - mv docker docker_${CI_BUILD_REF_SLUG}
+    - cd docker_${CI_BUILD_REF_SLUG}
+    - TMP_DOCKER=$(tempfile)
+    - cp docker-compose.yml $TMP_DOCKER
+    - awk -f ci/for-review.awk $TMP_DOCKER > docker-compose.yml
+    - sed -i '$e cat ci/fuse-entrypoint.sh' vidjil-server/scripts/fuse-entrypoint.sh
+    - rm -f $TMP_DOCKER
+    - sed -i -r "s/WEB2PY_ADMIN_PASSWORD=/WEB2PY_ADMIN_PASSWORD=$REVIEW_SERVER_ADMIN_PASSWORD/" docker-compose.yml
+    - cd ..
+    - rsync -av --exclude .git . $REVIEW_SERVER:$REVIEW_SERVER_PATH_SERVER$CI_BUILD_REF_SLUG || $(case "$?" in 0|23) exit 0 ;; *) exit $?; esac)
+    - ssh $REVIEW_SERVER "cd $REVIEW_SERVER_PATH_SERVER; ./install_certs.sh ${CI_BUILD_REF_SLUG}; cd ${CI_BUILD_REF_SLUG}/docker_${CI_BUILD_REF_SLUG}; ls -lnd ../browser | cut -d ' ' -f 3 > ../uid; docker-compose up -d mysql fuse uwsgi nginx workers"
+  environment:
+    name: review_server/$CI_BUILD_REF_NAME
+    url: http://$CI_BUILD_REF_SLUG.server.ci.vidjil.org/browser/
+    on_stop: stop_deploy_review_server
+  only:
+    - /^feature-.*s.*\/.*$/
+    - /^hotfix-.*s.*\/.*$/
+
 deploy_docker:
   stage: deploy_prod
   script:
@@ -466,3 +492,15 @@ stop_deploy_review:
     - /^hotfix-.*c.*\/.*$/
   tags:
     - web
+
+stop_deploy_review_server:
+  stage: deploy_review
+  script:
+    - ssh $REVIEW_SERVER "set -o xtrace; docker-compose -f $REVIEW_SERVER_PATH_SERVER$CI_BUILD_REF_SLUG/docker_$CI_BUILD_REF_SLUG/docker-compose.yml down -v ; rm -rf $REVIEW_SERVER_PATH_SERVER$CI_BUILD_REF_SLUG; $REVIEW_SERVER_PATH_SERVER/uninstall_certs.sh $CI_BUILD_REF_SLUG"
+  when: manual
+  environment:
+    name: review_server/$CI_BUILD_REF_NAME
+    action: stop
+  only:
+    - /^feature-.*s.*\/.*$/
+    - /^hotfix-.*s.*\/.*$/

docker/ci/for-review.awk

+#!/bin/awk -f
+
+BEGIN{
+    slug=ENVIRON["CI_BUILD_REF_SLUG"]
+}
+/\/mnt\/upload\/uploads/ {
+    print "            - ./ci/uploads:/mnt/upload/uploads"
+    next
+}
+/\/mnt\/result\/results/ {
+    print "            - ./ci/result:/mnt/result"
+    next
+}
+/\/mnt\/result\/tmp/ {
+    next
+}
+after_ports{
+    if(/\s*-\s*"[^"]*"/)
+        next
+    else
+        after_ports=0
+}
+after_service{
+    printf "        "
+    print "network_mode: bridge"
+    after_service=0
+}    
+after_nginx{
+    printf "        "
+    print "expose:"
+    printf "            "
+    print "- 443"
+    printf "        "
+    print "environment:"
+    printf "            "
+    print "- VIRTUAL_HOST=" slug ".server.ci.vidjil.org"
+    printf "            "
+    print "- VIRTUAL_PORT=443"
+    printf "            "
+    print "- VIRTUAL_PROTO=https"
+    after_nginx=0
+}
+after_volumes{
+  printf "            "
+  print "- ../browser:/usr/share/vidjil/browser"
+  printf "            "
+  print "- ../server/web2py/applications/vidjil:/usr/share/vidjil/server/web2py/applications/vidjil"
+  printf "            "
+  print "- ./:/usr/share/vidjil/docker"
+  printf "            "
+  print "- databases:/usr/share/vidjil/server/web2py/applications/vidjil/databases"
+  
+  after_volumes=0
+}
+
+after_volumes2{
+    if(/\s*-\s*\/opt\/*/) {
+      next
+    } else {
+      after_volumes2=0
+    }
+}
+/volumes:/{
+    after_volumes=1
+    after_volumes2=1
+}
+/nginx:$/{
+    after_nginx=1
+}
+/ports:/{
+    after_ports=1
+    next
+}
+/^\s{3,6}(nginx|fuse|uwsgi|workers|mysql):$/{
+    after_service=1
+}
+/\/opt\/vidjil\/mysql/ {
+    # No volume for MySQL
+    next
+}
+/uwsgi-entrypoint.sh/{
+    print $0" --ci"
+    next
+}
+1
+END {
+  printf "volumes:\n    databases:\n"
+}

docker/ci/result/results/fused_file.fused_file.84ac5fad20e80a84.3030303030312d31382e6675736564.fused

+fused.vidjil
\ No newline at end of file

docker/ci/result/results/fused_file.fused_file.9847c4057e54221d.3030303030312d31392e6675736564.fused

+fused.vidjil
\ No newline at end of file

docker/ci/result/results/results_file.data_file.ba2b37c7245410bc.3030303030312e7669646a696c.txt

+/usr/share/vidjil/doc/analysis-example.vidjil
\ No newline at end of file

docker/ci/uploads/sequence_file.data_file.9b47359b198fca49.44656d6f2d5835.fa

+/usr/share/vidjil/demo/Demo-X5.fa
\ No newline at end of file

docker/docker-compose.yml

@@ -40,6 +40,8 @@ services:
             - ./vidjil-server/conf:/etc/vidjil
             - ./vidjil-server/scripts:/entrypoints
             - ../germline:/usr/share/vidjil/germline
+        extra_hosts:
+            - "fuse:127.0.0.1"
     nginx:
         image: vidjil/client:latest
         depends_on:
@@ -59,7 +61,7 @@ services:
 
     workers:
         image: vidjil/server:latest
-        command: /usr/local/bin/gosu www-data python /usr/share/vidjil/server/web2py/web2py.py -K vidjil,vidjil,vidjil
+        command: bash /entrypoints/workers-entrypoint.sh
         volumes_from:
             - uwsgi
         depends_on:

docker/vidjil-server/scripts/fuse-entrypoint.sh

 #!/bin/bash
+. $(dirname $0)/tools.sh
+user=$(get_user_of_results)
 
 cd /usr/share/vidjil/server
-/usr/local/bin/gosu www-data python fuse_server.py
+/usr/local/bin/gosu $user python fuse_server.py

docker/vidjil-server/scripts/tools.sh

+#!/bin/bash
+get_user_of_results() {
+    if [ -d /mnt/result/results ]; then
+        user=$(stat -c '%u' /mnt/result/results)
+    else
+        user=www-data
+    fi
+    echo $user
+}
+

docker/vidjil-server/scripts/uwsgi-entrypoint.sh

 #!/bin/bash
+. $(dirname $0)/tools.sh
 touch /var/vidjil/vidjil.log
 touch /var/vidjil/vidjil-debug.log
-if [ -d /mnt/result/results ]; then
-    user=$(stat -c '%u' /mnt/result/results)
-else
-    user=www-data
-fi
+user=$(get_user_of_results)
 chown -R $user /var/vidjil/vidjil*
 chown -R $user /usr/share/vidjil/server/web2py/applications/vidjil/databases
 
@@ -14,4 +11,15 @@ if [[ -n "$WEB2PY_ADMIN_PASSWORD" ]]; then
     python -c "from gluon.main import save_password; save_password('${WEB2PY_ADMIN_PASSWORD}', 443)"
 fi
 
-/usr/local/bin/gosu $user /usr/bin/uwsgi --ini /etc/uwsgi/apps-enabled/web2py.ini
+/usr/local/bin/gosu $user /usr/bin/uwsgi --ini /etc/uwsgi/apps-enabled/web2py.ini &
+
+if [ "$1" == "--ci" ]; then
+    cd /usr/share/vidjil/server/web2py/applications/vidjil/scripts
+    sleep 20
+    echo "Loading SQL"
+    /usr/local/bin/gosu $user bash launch_python_script.sh load-sql.py -n /usr/share/vidjil/docker/ci/ci.sql
+    echo "SQL loaded"
+    python /usr/share/vidjil/tools/fuse.py --output /mnt/result/results/fused.vidjil /usr/share/vidjil/doc/analysis-example.vidjil /usr/share/vidjil/doc/analysis-example.vidjil /usr/share/vidjil/doc/analysis-example.vidjil
+fi
+
+wait

docker/vidjil-server/scripts/workers-entrypoint.sh

+#!/bin/bash
+. $(dirname $0)/tools.sh
+user=$(get_user_of_results)
+/usr/local/bin/gosu $user python /usr/share/vidjil/server/web2py/web2py.py -K vidjil,vidjil,vidjil

server/web2py/applications/vidjil/scripts/load-sql.py

+import sys
+
+if len(sys.argv) > 1:
+    if sys.argv[1] == "-h":
+        print ('Usage: {} [-n] <file.sql>\n\nLoad <file.sql> in the database. If -n do it only when database is empty.'.format(sys.argv[0]))
+    else:
+        i=1
+        check_empty = False
+        if sys.argv[1] == "-n":
+            check_empty = True
+            i += 1
+        filename = sys.argv[i]
+        
+        if check_empty and db(db.auth_user).count() > 0:
+            print ("Database is not empty. Not executing the SQL")
+            exit(0)
+            
+        with open(filename, "r") as sql:
+            print ("Reading file {}".format(filename))
+            queries = sql.read()
+            db.executesql(queries)