Commit 170482e7 authored by Samir Noir's avatar Samir Noir 🧀
Browse files

Remove eventmachine and em-synchrony

em-synchrony doesn't work with Activerecord 6, this commit remove the
dependencies to em-synchrony and eventmachine, which are not actively
maintained.

This change the way of connecting to MySQL and PostreSQL databases, and
the way of making http requests to Grid'5000 as well.
parent a67b7076
......@@ -10,3 +10,4 @@ coverage
.vagrant
*~
lib/tasks/tunneling.rake
.byebug_history
......@@ -6,10 +6,7 @@ gem 'bootsnap', require: false
# jQuery is the default JavaScript library as from Rails 3.1
gem 'jquery-rails'
gem 'eventmachine'
gem 'rack-fiber_pool', '~> 0.9'
gem 'em-synchrony'
gem 'em-http-request'
gem "mysql2", '~> 0.5.3'
gem "ruby-mysql", :require => "mysql"
gem 'addressable', '~> 2.2'
......@@ -20,7 +17,6 @@ gem 'syslogger'
gem 'haml', '~> 5.1.2'
gem 'rack-jsonp'
gem 'pg'
gem 'em-postgresql-adapter', :git => 'https://github.com/grid5000/em-postgresql-adapter.git'
gem 'nokogiri'
gem 'sass-rails'
......@@ -47,4 +43,3 @@ group :test do
gem 'byebug'
end
GIT
remote: https://github.com/grid5000/em-postgresql-adapter.git
revision: 6b28d457889281e95ad103c10d751a3f769ad2f6
specs:
em-postgresql-adapter (0.3)
activerecord (>= 3.1.0)
eventmachine
pg (>= 0.8.0)
GEM
remote: https://rubygems.org/
specs:
......@@ -80,7 +71,6 @@ GEM
execjs
coffee-script-source (1.12.2)
concurrent-ruby (1.1.6)
cookiejar (0.3.3)
crack (0.4.3)
safe_yaml (~> 1.0.0)
crass (1.0.6)
......@@ -89,16 +79,6 @@ GEM
docile (1.3.2)
domain_name (0.5.20190701)
unf (>= 0.0.5, < 1.0.0)
em-http-request (1.1.5)
addressable (>= 2.3.4)
cookiejar (!= 0.3.1)
em-socksify (>= 0.3)
eventmachine (>= 1.0.3)
http_parser.rb (>= 0.6.0)
em-socksify (0.3.2)
eventmachine (>= 1.0.0.beta.4)
em-synchrony (1.0.6)
eventmachine (>= 1.0.0.beta.1)
erubi (1.9.0)
erubis (2.7.0)
eventmachine (1.2.7)
......@@ -123,7 +103,6 @@ GEM
http-accept (1.7.0)
http-cookie (1.0.3)
domain_name (~> 0.5)
http_parser.rb (0.6.0)
i18n (1.8.2)
concurrent-ruby (~> 1.0)
ipaddress (0.8.3)
......@@ -289,11 +268,7 @@ DEPENDENCIES
bootsnap
byebug
coffee-rails
em-http-request
em-postgresql-adapter!
em-synchrony
erubis (~> 2.7)
eventmachine
factory_bot_rails
gitlab-grit
haml (~> 5.1.2)
......
......@@ -104,12 +104,16 @@ class ApplicationController < ActionController::Base
# Allow the list of "non-error" http codes
allowed_status = [options[:is] || (200..299).to_a].flatten
status = http.response_header.status # get the status from the http response
status = http.code.to_i # get the status from the http response
# hack to make rspec tests working, indeed for a unknown reason, http.uri is
# nil when running the specs suite
http.uri = http.header['Location'] if http.uri.nil?
if status.between?(400, 599) # error status
# http.method always returns nil. Bug?
# msg = "#{http.method} #{http.uri} failed with status #{status}"
msg = "Request to #{http.last_effective_url.to_s} failed with status #{status}: #{http.response}"
msg = "Request to #{http.uri} failed with status #{status}: #{http.body}"
Rails.logger.error msg
end
......@@ -137,7 +141,7 @@ class ApplicationController < ActionController::Base
when 503
raise ServerUnavailable, msg
else
raise ServerError, "Request to #{http.last_effective_url.to_s} failed with status #{status}: #{http.response}"
raise ServerError, "Request to #{http.uri.to_s} failed with status #{status}: #{http}"
Rails.logger.error msg
end
......@@ -165,7 +169,7 @@ class ApplicationController < ActionController::Base
when 503
raise ServerUnavailable, msg
else
raise ServerError, "Request to #{http.last_effective_url.to_s} failed with unexpected status #{status}: #{http.response} ; could be a problem with our version of eventmachine not supporting IPv6, or TLS problems"
raise ServerError, "Request to #{http.uri.to_s} failed with unexpected status #{status}: #{http} ; could be a problem with our version of eventmachine not supporting IPv6, or TLS problems"
end
end
......
......@@ -149,7 +149,6 @@ class DeploymentsController < ApplicationController
# from the kadeploy-server, and update the <tt>result</tt> attribute if the
# deployment has finished.
def update
byebug
dpl = find_item(params[:id])
dpl.base_uri = api_path()
dpl.tls_options=tls_options_for(dpl.base_uri, :out)
......
......@@ -68,29 +68,26 @@ class JobsController < ApplicationController
job = OAR::Job.find(params[:id])
authorize!(job.user)
url = uri_to(
uri = URI(uri_to(
site_path(
params[:site_id]
)+"/internal/oarapi/jobs/#{params[:id]}.json",
:out
)
options=tls_options_for(url, :out)
http = EM::HttpRequest.new(url,{:tls => options}).delete(
:timeout => 5,
:head => {
'X-Remote-Ident' => @credentials[:cn],
'X-Api-User-Cn' => @credentials[:cn],
'Accept' => api_media_type(:json)
}
)
))
tls_options = tls_options_for(uri, :out)
headers = { 'Accept' => api_media_type(:json),
'X-Remote-Ident' => @credentials[:cn],
'X-Api-User-Cn' => @credentials[:cn],
}
http = http_request(:delete, uri, tls_options, 5, headers)
continue_if!(http, :is => [200,202,204,404])
if http.response_header.status == 404
if http.code.to_i == 404
raise NotFound, "Cannot find job##{params[:id]} on the OAR server"
else
response.headers['X-Oar-Info'] = (
JSON.parse(http.response)['oardel_output'] || ""
response.header['X-Oar-Info'] = (
JSON.parse(http.body)['oardel_output'] || ""
).split("\n").join(" ") rescue "-"
location_uri = uri_to(
......@@ -117,22 +114,21 @@ class JobsController < ApplicationController
job_to_send = job.to_hash(:destination => "oar-2.4-submission")
Rails.logger.info "Submitting #{job_to_send.inspect}"
url = uri_to(
uri = URI(uri_to(
site_path(params[:site_id])+"/internal/oarapi/jobs.json", :out
)
options=tls_options_for(url, :out)
http = EM::HttpRequest.new(url, {:tls => options}).post(
:timeout => 20,
:body => job_to_send.to_json,
:head => {
'X-Remote-Ident' => @credentials[:cn],
'X-Api-User-Cn' => @credentials[:cn],
'Content-Type' => api_media_type(:json),
'Accept' => api_media_type(:json)
} )
))
tls_options = tls_options_for(uri, :out)
headers = { 'X-Remote-Ident' => @credentials[:cn],
'X-Api-User-Cn' => @credentials[:cn],
'Content-Type' => api_media_type(:json),
'Accept' => api_media_type(:json)
}
http = http_request(:post, uri, tls_options, 5, headers, job_to_send.to_json)
continue_if!(http, :is => [201,202])
job_uid = JSON.parse(http.response)['id']
job_uid = JSON.parse(http.body)['id']
location_uri = uri_to(
resource_path(job_uid),
:in, :absolute
......@@ -140,7 +136,7 @@ class JobsController < ApplicationController
job = OAR::Job.expanded.includes(:job_types, :job_events, :gantt).find(job_uid)
job.links = links_for_item(job)
render_opts = {
:methods => [:resources_by_type, :assigned_nodes],
:location => location_uri,
......
......@@ -29,11 +29,11 @@ class ResourcesController < ApplicationController
def fetch(path)
allow :get; vary_on :accept
Rails.logger.info "Fetching #{path}"
enrich_params(params)
object=lookup_path(path,params)
object=lookup_path(path,params)
raise NotFound, "Cannot find resource #{path}" if object.nil?
if object.has_key?('items')
......@@ -85,39 +85,39 @@ class ResourcesController < ApplicationController
# abasu : In request to feature bug ref. #6363
# abasu : params_queues is the array with 'queues' values passed in 'params'
if params[:queues].nil? # no filter, so assign everything except "production"
params[:queues] = ["admin","default"]
params[:queues] = ["admin","default"]
# As of 11.12.2015 the queues accepted are:
# "all" or any combination of "admin", "default", "production"
else
else
if params[:queues] == "all" # for use by sys-admin
params[:queues] = ["admin","default","production"]
else
params[:queues] = params[:queues].split(",")
end # if params[:queues] == "all"
end # if params[:queues].nil?
end
def lookup_path(path, params)
object = EM::Synchrony.sync repository.async_find(
object = repository.find(
path.gsub(/\/?platforms/,''),
:branch => params[:branch],
:version => params[:version]
)
raise ServerUnavailable if object.is_a?(Exception)
# abasu : case logic for treating different scenarios - 11.12.2015
case [params[:controller], params[:action]]
# 1. case of a single cluster
when ["clusters", "show"]
when ["clusters", "show"]
# Add ["admin","default"] to 'queues' if nothing defined for that cluster
object['queues'] = ["admin","default"] if object['queues'].nil?
object = nil if (object['queues'] & params[:queues]).empty?
# 2. case of an array of clusters
when ["clusters", "index"]
when ["clusters", "index"]
# First, add ["admin","default"] to 'queues' if nothing defined for that cluster
object['items'].each { |cluster| cluster['queues'] = ["admin","default"] if cluster['queues'].nil? }
# Then, filter out 'queues' that are not requested in params
......@@ -129,11 +129,11 @@ class ResourcesController < ApplicationController
=end
# Finally, set new 'total' to clusters shortlisted
object['total'] = object['items'].length
end # case [params[:controller], params[:action]]
object
end
# Must be overwritten by descendants
def collection_path
raise NotImplemented
......@@ -154,8 +154,8 @@ class ResourcesController < ApplicationController
(item.delete('subresources') || []).each do |subresource|
href = uri_to(resource_path(item["uid"]) + "/" + subresource.name)
links.push({
"rel" => subresource.name,
"href" => href,
"rel" => subresource.name,
"href" => href,
"type" => api_media_type(:g5kcollectionjson)
})
end
......
......@@ -16,12 +16,12 @@ require 'grid5000/repository'
require 'grid5000/router'
module ApplicationHelper
def link_attributes_for(attributes = {})
attributes[:type] ||= default_media_type
attributes
end
def uri_to(path, in_or_out = :in, relative_or_absolute = :relative)
Grid5000::Router.uri_to(request, path, in_or_out, relative_or_absolute)
end
......@@ -35,7 +35,7 @@ module ApplicationHelper
File.expand_path(
Rails.my_config(:reference_repository_path),
Rails.root
),
),
Rails.my_config(:reference_repository_path_prefix),
Rails.logger
)
......@@ -48,4 +48,29 @@ module ApplicationHelper
end
end
def http_request(method, uri, tls_options, timeout = nil, headers = {}, body = nil)
http = Net::HTTP.new(uri.host, uri.port)
http.read_timeout = 5 if timeout
http.use_ssl = true if uri.scheme == 'https'
if tls_options && !tls_options.empty?
http.cert = OpenSSL::X509::Certificate.new(File.read(tls_options[:cert_chain_file]))
http.key = OpenSSL::PKey::RSA.new(File.read(tls_options[:private_key_file]))
http.verify_mode = tls_options[:verify_peer]
end
request = case method
when :post
Net::HTTP::Post.new(uri, headers)
when :get
Net::HTTP::Get.new(uri, headers)
when :delete
Net::HTTP::Delete.new(uri, headers)
else
raise "Unknown http method: #{method}"
end
request.body = body if body
http.request(request)
end
end
......@@ -127,14 +127,14 @@ module Grid5000
def cancel_workflow!
raise "cancel_workflow!" if !user or !base_uri # Ugly hack
connect_options={:timeout => 15,:tls => tls_options}
http = EM::HttpRequest.new(File.join(base_uri,uid), connect_options).delete(
:head => {
#'Accept' => '*/*',
'X-Remote-Ident' => user,
}
)
http.errback{ error("Unable to contact #{File.join(base_uri,uid)}"); raise self.output+"\n" }
headers = { 'X-Remote-Ident' => user }
uri = base_uri + uid
response = http_request(:delete, uri, tls_options, 15, headers, body.to_s)
unless %w{200 201 202 204}.include?(response.status.to_s)
error("Unable to contact #{File.join(base_uri,uid)}")
raise self.output+"\n"
end
# Not checked since it avoid touch! to cancel the deployment
#unless %w{200 201 202 204}.include?(http.response_header.status.to_s)
......@@ -155,9 +155,8 @@ module Grid5000
case scheme
when 'http','https'
begin
connect_options={:timeout => 10,:tls => tls_options}
http = EM::HttpRequest.new(environment, connect_options).get()
params['environment'] = YAML.load(http.response)
http = http_request(:get, environment, tls_options, 10, headers)
params['environment'] = YAML.load(http.body)
params['environment']['kind'] = 'anonymous'
rescue Exception => e
raise "Error fetching the image description file: #{e.class.name}, #{e.message}"
......@@ -170,21 +169,21 @@ module Grid5000
end
Rails.logger.info "Submitting: #{params.inspect} to #{base_uri}"
connect_options={:timeout => 20,:tls => tls_options}
http = EM::HttpRequest.new(base_uri, connect_options).post(
:body => params.to_json,
:head => {
'Content-Type' => Mime::Type.lookup_by_extension(:json).to_s,
'Accept' => Mime::Type.lookup_by_extension(:json).to_s,
'X-Remote-Ident' => user,
}
)
http.errback{ error("Unable to contact #{base_uri}"); raise self.output+"\n" }
if %w{200 201 202 204}.include?(http.response_header.status.to_s)
update_attribute(:uid, JSON.parse(http.response)['wid'])
headers = { 'Content-Type' => Mime::Type.lookup_by_extension(:json).to_s,
'Accept' => Mime::Type.lookup_by_extension,
'X-Remote-Ident' => user,
}
http = http_request(:post, base_uri, tls_options, 20, headers)
unless %w{200 201 202 204}.include?(http.status.to_s)
error("Unable to contact #{base_uri}")
raise self.output+"\n"
end
if %w{200 201 202 204}.include?(http.status.to_s)
update_attribute(:uid, JSON.parse(http.body)['wid'])
else
error(get_kaerror(http.response,http.response_header))
error(get_kaerror(http,http.headers))
# Cannot continue since :uid is not set
raise self.output+"\n"
end
......@@ -193,43 +192,49 @@ module Grid5000
end
def touch!
connect_options={:timeout => 10,:tls => tls_options}
http = EM::HttpRequest.new(File.join(base_uri,uid), connect_options).get(
:head => {
'Accept' => Mime::Type.lookup_by_extension(:json).to_s,
'X-Remote-Ident' => user,
}
)
http.errback{ error("Unable to contact #{File.join(base_uri,uid)}"); raise self.output+"\n" }
if %w{200 201 202 204}.include?(http.response_header.status.to_s)
item = JSON.parse(http.response)
begin
header = { 'Accept' => Mime::Type.lookup_by_extension(:json).to_s,
'X-Remote-Ident' => user
}
uri = base_uri + uid
http = http_request(:get, uri, tls_options, 10, headers)
rescue
error("Unable to contact #{File.join(base_uri,uid)}")
raise self.output+"\n"
end
if %w{200 201 202 204}.include?(http.status.to_s)
item = JSON.parse(http.body)
unless item['error']
connect_options={:timeout => 15,:tls => tls_options}
http = EM::HttpRequest.new(File.join(base_uri,uid,'state'), connect_options).get(
:head => {
'Accept' => Mime::Type.lookup_by_extension(:json).to_s,
'X-Remote-Ident' => user,
}
)
http.errback{ error("Unable to contact #{File.join(base_uri,uid,'state')}"); raise self.output+"\n" }
res = JSON.parse(http.response)
begin
header = { 'Accept' => Mime::Type.lookup_by_extension(:json).to_s,
'X-Remote-Ident' => user
}
uri = base_uir + uid + 'state'
http = http_request(:get, uri, tls_options, 15, headers)
rescue
error("Unable to contact #{File.join(base_uri,uid, 'state')}")
raise self.output+"\n"
end
res = JSON.parse(http.body)
# Ugly compatibility hack
res.each_pair do |node,stat|
res[node]['state'] = res[node]['state'].upcase
end
self.result = res
else
connect_options={:timeout => 15,:tls => tls_options}
http = EM::HttpRequest.new(File.join(base_uri,uid,'error'), connect_options).get(
:head => {
#'Accept' => '*/*',
'X-Remote-Ident' => user,
}
)
error(get_kaerror(http.response,http.response_header))
http.errback{ error("Unable to contact #{File.join(base_uri,uid,'error')}"); raise self.output+"\n" }
begin
headers = { 'X-Remote-Ident' => user }
uri = base_uri + uid + 'error'
http = http_request(:get, uri, tls_options, 15, headers)
rescue
error(get_kaerror(http,http.headers))
error("Unable to contact #{File.join(base_uri,uid,'error')}")
raise self.output + "\n"
end
return
end
......
......@@ -66,15 +66,12 @@ class Notification
case uri.scheme
# HTTP processing
when /http/
http = EM::HttpRequest.new(uri.to_s).post(
:timeout => 5,
:body => body.to_s,
:head => {
'Content-Type' => "text/plain",
'Accept' => "*/*"
}
)
Rails.logger.info "Sent notification, received status=#{http.response_header.status}: #{http.response.inspect}"
headers = { 'Content-Type' => 'text/plain', 'Accept' => '*/*' }
request = Net::Http::Post.new(uri, headers)
request.body = body.to_s
response = http.request(request)
Rails.logger.info "Sent notification, received status=#{response.status}: #{response.inspect}"
# EMAIL processing
when /mailto/
subject = uri.headers.detect{|array| array.first == "subject"}
......@@ -101,4 +98,4 @@ class Notification
Rails.logger.warn "Failed to send notification #{self.inspect} : #{e.class.name} - #{e.message}"
Rails.logger.debug e.backtrace.join(";")
end
end
\ No newline at end of file
end
......@@ -25,14 +25,14 @@ if defined?(Bundler)
end
# Explicitly require libs when gem name is not sufficient
require 'em-synchrony'
require 'em-synchrony/em-http'
require 'em-http'
require 'addressable/uri'
require 'rack/fiber_pool'
require 'rack/jsonp'
require 'rack/lint'
# Use net/http to contact other g5k's services
require "net/http"
module Api
class Application < Rails::Application
......
......@@ -15,7 +15,7 @@ defaults: &defaults
smtp_port: 25
smtp_from: api@grid5000.fr
oar: &oar
adapter: em_postgresql
adapter: postgresql
encoding: utf8
pool: 20
timeout: 5000
......@@ -85,7 +85,7 @@ test_mysql:
<<: *test
oar:
<<: *oar
adapter: em_mysql2
adapter: mysql2
host: 127.0.0.1
port: 13306
username: root
......
......@@ -128,8 +128,9 @@ describe JobsController do
:body => Grid5000::Job.new(payload).to_hash(:destination => "oar-2.4-submission").to_json
).
to_return(
:status => 400,
:body => "some error"
:headers => { 'Location' => expected_url },
:status => [400, 'Bad Request'],
:body => 'some error',
)
post :create, params: { :site_id => "rennes", :format => :json }, body: payload.to_json, as: :json
......@@ -153,8 +154,9 @@ describe JobsController do
:body => Grid5000::Job.new(payload).to_hash(:destination => "oar-2.4-submission").to_json
).
to_return(
:status => 400,
:body => "Bad Request"
:status => [400, 'Bad Request'],
:body => 'Bad Request',
:headers => { 'Location' => expected_url },
)
post :create, params: { :site_id => "rennes", :format => :json }, body: payload.to_json, as: :json