Commit ce6b6423 authored by Samir Noir's avatar Samir Noir 🧀
Browse files

Merge branch 'buster' into 'master'

Buster support

See merge request !62
parents b55c8fd6 2608d249
Pipeline #138917 waiting for manual action with stages
in 23 minutes and 10 seconds
......@@ -39,14 +39,14 @@ test-for-jessie: &test-for-jessie
except:
- tags
test-for-stretch:
test-for-buster:
<<: *test-for-jessie
image: debian:stretch
image: debian:buster
services:
- name: mariadb:10.1
- name: mariadb:10.3
alias: mysql
- name: postgres
deb-for-jessie: &deb-for-jessie
image: debian:jessie
stage: deb
......@@ -56,24 +56,18 @@ deb-for-jessie: &deb-for-jessie
- apt-get update && apt-get -y --no-install-recommends install devscripts build-essential equivs
- mk-build-deps -ir -t 'apt-get -y --no-install-recommends'
- apt-get install -y curl pkg-config
- bundle install --path vendor/bundle
- bundle exec rake package:build:debian
- dpkg-buildpackage
- cp ../*deb .
artifacts:
paths:
- pkg/g5k-api*.deb
- ./g5k-api*.deb
expire_in: '1 day'
except:
- tags
deb-for-stretch:
deb-for-buster:
<<: *deb-for-jessie
image: debian:stretch
script:
- apt-get update && apt-get -y --no-install-recommends install devscripts build-essential equivs
- mk-build-deps -ir -t 'apt-get -y --no-install-recommends'
- apt-get install -y curl pkg-config
- bundle install --path vendor/bundle
- bundle exec rake package:build:debian
image: debian:buster
.push-packages-script: &push-packages-script |
echo "Request to push for debian version ${TARGET_DEBIAN_VERSION} for branch ${TARGET_G5K_API_VARIANT}"
......@@ -101,38 +95,46 @@ push-jessie-package-v4: &push-jessie-package-v4
variables:
TARGET_DEBIAN_VERSION: jessie
TARGET_G5K_API_VARIANT: v4
only:
only:
- devel
push-jessie-package-devel: &push-jessie-package-devel
<<: *push-jessie-package-v3
variables:
TARGET_DEBIAN_VERSION: jessie
TARGET_G5K_API_VARIANT: devel
only:
only:
- devel
push-stretch-package-v3:
<<: *push-jessie-package-v3
push-buster-package-v3: &push-buster-package-v3
stage: deploy
when: manual
allow_failure: false
# tags must be 'packages' so that we use the runner on packages.grid5000.fr
tags:
- grid5000-packages
only: # we only execute this job for tagged commits on the master branch
- master
dependencies:
- deb-for-stretch
- deb-for-buster
variables:
TARGET_DEBIAN_VERSION: stretch
TARGET_DEBIAN_VERSION: buster
TARGET_G5K_API_VARIANT: v3
script:
- *push-packages-script
push-stretch-package-v4:
<<: *push-jessie-package-v4
dependencies:
- deb-for-stretch
push-buster-package-v4: &push-buster-package-v4
<<: *push-buster-package-v3
variables:
TARGET_DEBIAN_VERSION: stretch
TARGET_DEBIAN_VERSION: buster
TARGET_G5K_API_VARIANT: v4
push-stretch-package-devel:
<<: *push-jessie-package-devel
dependencies:
- deb-for-stretch
only:
- devel
push-buster-package-devel: &push-buster-package-devel
<<: *push-buster-package-v3
variables:
TARGET_DEBIAN_VERSION: stretch
TARGET_DEBIAN_VERSION: buster
TARGET_G5K_API_VARIANT: devel
only:
- devel
......@@ -9,7 +9,7 @@ gem 'eventmachine'
gem 'rack-fiber_pool', '~> 0.9'
gem 'em-synchrony'
gem 'em-http-request'
gem "mysql2", '~> 0.3.6'
gem "mysql2", '~> 0.4.0'
gem "ruby-mysql", :require => "mysql"
gem 'addressable', '~> 2.2'
gem 'thin', '~> 1.5.0'
......
......@@ -117,7 +117,7 @@ GEM
mini_mime (1.0.1)
mini_portile2 (2.3.0)
minitest (5.11.3)
mysql2 (0.3.21)
mysql2 (0.4.10)
net-ssh (4.2.0)
net-ssh-gateway (2.0.0)
net-ssh (>= 4.0.0)
......@@ -262,7 +262,7 @@ DEPENDENCIES
gitlab-grit
haml (~> 4.0.4)
jquery-rails
mysql2 (~> 0.3.6)
mysql2 (~> 0.4.0)
net-ssh (< 5.0.0)
net-ssh-multi
nokogiri
......
......@@ -39,14 +39,14 @@ You can check the installation by running a few commands to gather information a
### Development environment
* This app comes with a Vagrant box used for development, testing and packaging.
* This app comes with a Vagrant box used for development, testing and packaging.
By default, the vagrant box will provision a proxy, to get access to the live status
of sites and to the home directory of users, except on one site where status will be
served locally through a tunnel to that site's oardb. This is specially useful to
debug the web ui, but the tunnel to the db is also used for site status information.
debug the web ui, but the tunnel to the db is also used for site status information.
For users with a working installation of vagrant and virtualbox, setting up a
For users with a working installation of vagrant and virtualbox, setting up a
working environement starts with a simple
$ DEVELOPER=dmargery OAR_DB_SITE=rennes vagrant up --provision
......@@ -54,15 +54,15 @@ You can check the installation by running a few commands to gather information a
vagrant> cd /vagrant
The vagrant provisionning script will attempt to configure the VM's root and vagrant
accounts to be accessible by ssh. By default, it will copy your authorized_keys, but you
can control the keypair used with SSH_KEY=filename_of_private_key
accounts to be accessible by ssh. By default, it will copy your authorized_keys, but you
can control the keypair used with SSH_KEY=filename_of_private_key
Of course, reality is a bit more complex. You might have troubles with the insecure
certificate of the vagrant box provider. In that case, you'll need to start with
certificate of the vagrant box provider. In that case, you'll need to start with
$ vagrant box add --insecure --name debian-jessie-x64-puppet_4 \
https://vagrant.irisa.fr/boxes/irisa_debian-8.2.0_puppet4.box
And as the application relies on external data sources, you'll need to connect
it with a reference-repository, an OAR database, a kadeploy3 server, and a jabber server
to exercice all its functionnality, in addition to its own backend services that
......@@ -70,14 +70,14 @@ You can check the installation by running a few commands to gather information a
the development of this application relies strongly on unit tests.
A useful set of ssh tunnels is created with
vagrant> rake tunnels:setup
vagrant> bundle exec rake tunnels:setup
### Development environment's access to data
* Setup the database schema:
vagrant> rake db:setup RAILS_ENV=development
vagrant> bundle exec rake db:setup RAILS_ENV=development
* Give access to reference data to expose. The scripts expect you
have a checkout version of the reference-repository in a sibling directory
......@@ -86,10 +86,10 @@ You can check the installation by running a few commands to gather information a
(g5k-api) $ cd ..
(..) $ git clone ssh://g5kadmin@git.grid5000.fr/srv/git/repos/reference-repository.git reference-repository
You might not have admin access to Grid'5000's reference repository. in this case, you
could duplicate the fake repository used for tests, in the
spec/fixtures/reference-repository directory.
You might not have admin access to Grid'5000's reference repository. in this case, you
could duplicate the fake repository used for tests, in the
spec/fixtures/reference-repository directory.
(g5k-api) $ cp -r spec/fixtures/reference-repository ..
(g5k-api) $ mv ../reference-repository/git.rename ../reference-repository/.git
......@@ -97,23 +97,23 @@ You can check the installation by running a few commands to gather information a
Do not attempt to use the directory directly, as unit test play with the git.rename dir.
* Get access to a OAR database, unsing one of the two methods described hereafter:
* Get your hands on a copy of an active database, and install it. Don't worry about the
error messages when seeding the development database: most of them come from the fact
that the database is empty and therefore the drop statements fail.
$ export OAR_DB_SITE=rennes
$ ssh oardb.$OAR_DB_SITE.g5kadmin sudo cat /var/backups/dumps/postgresql/oar2.sql.gz > tmp/oar2_$OAR_DB_SITE.sql.gz
$ gunzip tmp/oar2_$OAR_DB_SITE.sql.gz
vagrant>SEED=tmp/oar2_rennes.sql RAILS_ENV=development rake db:oar:seed
vagrant>SEED=tmp/oar2_rennes.sql RAILS_ENV=development bundle exec rake db:oar:seed
* Or tunnel your way to a live database (as g5k-api only requires read-only access)
This is particularly usefull if you want to develop on the UI (but with bad site
information). You should setup an SSH tunnel between your machine and one of the
This is particularly usefull if you want to develop on the UI (but with bad site
information). You should setup an SSH tunnel between your machine and one of the
oardb servers of Grid'5000, so that you can access the current jobs:
$ # In an other shell, create a tunnel from the vagrant machine to Grid'5000
$ # (done as part of rake tunnels:setup)
$ # (done as part of bundle exec rake tunnels:setup)
vagrant> ssh -NL 15433:oardb.rennes.grid5000.fr:5432 access.grid5000.fr
$ # edit the development section of config/defaults.yml to
......@@ -124,7 +124,7 @@ You can check the installation by running a few commands to gather information a
username: oarreader
password: read
database: oar2
### Wrapping it up to run the server
......@@ -140,12 +140,12 @@ You can check the installation by running a few commands to gather information a
$ HTTP_X_API_USER_CN=dmargery WHOAMI=rennes ./bin/g5k-api server start -e development
* If you want to develop on the UI, using the apache proxy, run your browser on
* If you want to develop on the UI, using the apache proxy, run your browser on
$ firefox http://127.0.0.1:8080/ui
* If you want to develop on the UI, interacting directly with the server, run your browser on
* If you want to develop on the UI, interacting directly with the server, run your browser on
$ firefox http://127.0.0.1:8000/ui
That's it. If you're not too familiar with `rails` 4, have a look at
......@@ -154,7 +154,7 @@ That's it. If you're not too familiar with `rails` 4, have a look at
You can also list the available rake tasks and capistrano tasks to see what's
already automated for you:
$ rake -T
$ bundle exec rake -T
$ cap -T
......@@ -165,16 +165,16 @@ to create a test database, and a fake OAR database.
* Create the test database as follows:
$ RAILS_ENV=test rake db:setup
$ RAILS_ENV=test bundle exec rake db:setup
* Create the fake OAR database as follows:
$ RAILS_ENV=test rake db:oar:create # provisioned by puppet
$ RAILS_ENV=test rake db:oar:seed
$ RAILS_ENV=test bundle exec rake db:oar:create # provisioned by puppet
$ RAILS_ENV=test bundle exec rake db:oar:seed
* Launch the tests:
$ RAILS_ENV=test rspec spec/ # or `bundle exec autotest` if rake fails (issue with Rails<3.1 and em_mysql2 adapter with evented code).
$ RAILS_ENV=test bundle exec rspec spec/ # or `bundle exec autotest` if rake fails (issue with Rails<3.1 and em_mysql2 adapter with evented code).
* Adding data to the reference repository used for tests
......@@ -198,12 +198,12 @@ After these changes, a lot of tests will fail as they rely on the hash of the la
* Adding data to the test OAR2 database
The test OAR2 database is loaded in the test environment using the `RAILS_ENV=test rake db:oar:seed` task defined in `lib/tasks/test.rake`. This tasks loads a SQL dump pointed by the SEED environment variable, and defaults to `spec/fixtures/oar2_2011-01-07.sql`
The test OAR2 database is loaded in the test environment using the `RAILS_ENV=test bundle exec rake db:oar:seed` task defined in `lib/tasks/test.rake`. This tasks loads a SQL dump pointed by the SEED environment variable, and defaults to `spec/fixtures/oar2_2011-01-07.sql`
Updating the OAR2 test db therefore requires either
* Updating the contents of the `spec/fixtures/oar2_2011-01-07.sql` directly
* creating a new dump of the `oar2_test` database running on the vagrant box, and updating `lib/tasks/test.rake` to use that dump a the new seed (be sure to git add it to the repo). As that `oar2_test` database should be a valid oar2 database, its contents should be updatable using oar commands. You'll need to apt-get install oar2 first and configure `oar2.conf` to point to the test database first. If you do so, please consider spending a few minutes to update the puppet provisonning recipes so as to pre-configure the vagrant VM to manage the test database out of the box. Furture developper will thank you for that.
* creating a new dump of the `oar2_test` database running on the vagrant box, and updating `lib/tasks/test.rake` to use that dump a the new seed (be sure to git add it to the repo). As that `oar2_test` database should be a valid oar2 database, its contents should be updatable using oar commands. You'll need to apt-get install oar2 first and configure `oar2.conf` to point to the test database first. If you do so, please consider spending a few minutes to update the puppet provisonning recipes so as to pre-configure the vagrant VM to manage the test database out of the box. Furture developper will thank you for that.
## Packaging
......@@ -225,11 +225,11 @@ For this to work properly, you need a working .gitconfig.
- You can now name the version you are about to package
vagrant@g5k-local: rake package:bump:patch #replace patch by minor or major when appropriate)
vagrant@g5k-local: bundle exec rake package:bump:patch #replace patch by minor or major when appropriate)
- And then build the debian package
vagrant@g5k-local: rake package:build:debian
vagrant@g5k-local: bundle exec rake package:build:debian
The `package:build:debian` rake task has several arguments:
......@@ -288,7 +288,7 @@ From time to time, someone will have to look into `lib/taks/packaging.rake` to u
* Generate the test coverage:
$ RAILS_ENV=test rspec spec/
$ RAILS_ENV=test bundle exec rspec spec/
$ open coverage/index.html
......
......@@ -13,7 +13,7 @@ Vagrant.configure(VAGRANTFILE_API_VERSION) do |config|
# please see the online documentation at vagrantup.com.
# Every Vagrant virtual environment requires a box to build off of.
config.vm.box = "debian/contrib-jessie64"
config.vm.box = "debian/contrib-buster64"
config.vm.hostname = "g5k-local"
# Create a forwarded port mapping which allows access to a specific port
......@@ -85,7 +85,7 @@ Vagrant.configure(VAGRANTFILE_API_VERSION) do |config|
if ! which pupppet > /dev/null ; then
sed -i s/httpredir/deb/ /etc/apt/sources.list # faster mirror
export DEBIAN_FRONTEND=noninteractive
cd /tmp && wget -q http://apt.puppetlabs.com/puppetlabs-release-pc1-jessie.deb && dpkg -i puppetlabs-release-pc1-jessie.deb
cd /tmp && wget -q http://apt.puppetlabs.com/puppet6-release-buster.deb && dpkg -i puppet6-release-buster.deb
apt-get update
apt-get -y install --no-install-recommends puppet-agent
fi
......
......@@ -12,7 +12,7 @@ development:
adapter: em_mysql2
encoding: utf8
database: g5kapi_dev
pool: 100
pool: 50
timeout: 5000
username: root
password:
......
......@@ -71,7 +71,7 @@ module Grid5000
if root_uri.path.blank?
root_path=''
else
root_path=root_uri.path+'/'
root_path=root_uri.path
end # if root_uri.path.blank?
uri = URI.join(root_uri, root_path+uri).to_s
......
# This puppet manifest is just here to configure a vanilla squeeze with the
# This puppet manifest is just here to configure a vanilla buster with the
# required libs and configuration to serve as a development machine for the
# g5k-api software. Production recipes are in the Grid'5000 puppet repository.
class development {
......@@ -10,7 +10,7 @@ class development {
include git
include dpkg::dev
include apache
postgres::database {
'oar2_dev':
ensure => present;
......@@ -42,11 +42,18 @@ class development {
require => Exec["allow connections to postgres for oar"]
}
exec{ "allow connections to postgres for root":
user => postgres,
command => "/bin/echo \"CREATE ROLE root LOGIN; GRANT ALL PRIVILEGES ON *.oar2_dev TO 'root' ;GRANT ALL PRIVILEGES ON *.oar2_test TO 'root' ;\" | /usr/bin/psql ",
unless => "/usr/bin/psql -l | grep root",
require => Service["postgresql"]
}
mysql::database {
'g5kapi_dev':
ensure => present;
}
mysql::database {
'g5kapi_test':
ensure => present;
......@@ -56,12 +63,12 @@ class development {
user => root,
group => root,
cwd => $workspace,
command => "/bin/su -c '/usr/local/bin/bundle install' $owner",
require => [Exec["install bundler"],Package['libxml2-dev','libxslt-dev']],
command => "/bin/su -c 'bundle install' $owner",
require => [Package['bundler'],Package['libxml2-dev','libxslt-dev']],
logoutput => true,
unless => "/bin/su -c '/usr/local/bin/bundle show' $owner"
unless => "/bin/su -c 'bundle show' $owner"
}
# Because of the way access to mysql and postgres work in
# docker for gitlab.inria.fr
exec { "Make sure mysql and postgres resolve to 127.0.0.1":
......@@ -91,9 +98,8 @@ class development {
stage { "init": before => Stage["main"] }
class {"apt":
class {"apt":
stage => init,
}
include development
class mysql {
package{"mysql-server":
package{"default-mysql-server":
ensure => latest
}
service{ "mysql":
ensure => running,
require => Package["mysql-server"]
require => Package["default-mysql-server"]
}
exec{ "set an empty root mysql password":
user => root, group => root,
command => "/usr/bin/mysql -e \"SET PASSWORD FOR 'root'@'localhost' = PASSWORD('');\"",
require => Service["mysql"]
}
exec{ "allow mysql connections from all":
user => root, group => root,
command => "/usr/bin/mysql -e \"GRANT ALL PRIVILEGES ON *.* TO 'root'@'%';\"",
command => "/usr/bin/mysql -e \"GRANT ALL PRIVILEGES ON *.* TO 'root'@'localhost' IDENTIFIED via mysql_native_password;\"",
require => Service["mysql"]
}
......@@ -20,10 +26,16 @@ class mysql {
require => Service["mysql"]
}
exec{ "flush privileges":
user => root, group => root,
command => "/usr/bin/mysql -e \"FLUSH PRIVILEGES;\"",
require => Service["mysql"]
}
file{ "/etc/mysql/conf.d/custom.cnf":
mode => "0644", owner => root, group => root,
ensure => file,
require => Package["mysql-server"],
require => Package["default-mysql-server"],
content => "[mysqld]\nbind-address = 0.0.0.0\nport = 3306\n",
notify => Service["mysql"]
}
......
......@@ -12,14 +12,14 @@ class postgres {
}
file{
'/etc/postgresql/9.4/main':
'/etc/postgresql/11/main':
ensure => directory,
owner => postgres,
group => postgres,
mode => "0700",
recurse => true,
require => [Package['postgresql'], User['postgres']];
'/etc/postgresql/9.4/main/pg_hba.conf':
'/etc/postgresql/11/main/pg_hba.conf':
ensure => present,
source => "puppet:///modules/postgres/pg_hba.conf",
owner => postgres,
......@@ -27,7 +27,7 @@ class postgres {
mode => "0700",
require => Package['postgresql'],
notify => Service['postgresql'];
'/etc/postgresql/9.4/main/postgresql.conf':
'/etc/postgresql/11/main/postgresql.conf':
content => template('postgres/postgres.conf.erb'),
owner => postgres,
group => postgres,
......
......@@ -5,11 +5,11 @@ class rails {
'libxml2-dev', #nokogiri
'libxslt-dev', #nokogiri
'libicu-dev', #charlock_holmes
'libmysqlclient-dev', #mysql2
'default-libmysqlclient-dev', #mysql2
'libpq-dev', #pg
'nodejs', #rails >= 3.1
]:
ensure => installed
}
} #class rails
\ No newline at end of file
} #class rails
class ruby {
package{"ruby":
ensure => latest
}
package{"ruby-dev":
ensure => latest
}
exec { "install rake":
user => root, group => root,
command => "/usr/bin/gem install --no-ri --no-rdoc rake",
require => Package["ruby"],
creates => "/usr/local/bin/rake"
}
$ruby_packages = ['ruby', 'ruby-dev', 'rake', 'bundler']
exec { "install bundler":
user => root, group => root,
# bundler version must be less than 2.0.0 until running with ruby version < 2.3.0 as with debian jessie
command => "/usr/bin/gem install --no-ri --no-rdoc bundler --version '< 2.0.0'",
require => Package["ruby"],
creates => "/usr/local/bin/bundle"
package{$ruby_packages:
ensure => latest
}
}
class ruby {
package{"ruby1.9.1-full":
ensure => latest
}
exec { "install rake":
user => root, group => root,
command => "/usr/bin/gem1.9.1 install --no-ri --no-rdoc rake --version 10.0.2",
require => Package["ruby1.9.1-full"],
creates => "/var/lib/gems/1.9.1/bin/rake"
}
exec { "install bundler":
user => root, group => root,
command => "/usr/bin/gem1.9.1 install --no-ri --no-rdoc bundler --version 1.7.6",
require => Package["ruby1.9.1-full"],
creates => "/var/lib/gems/1.9.1/bin/bundle"
}
}
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment