Commit 6319e922 authored by Samir Noir's avatar Samir Noir 🧀
Browse files

Merge branch 'features/#12051_deep' into 'master'

Add the ability to return a deep view of reference-repository

See merge request !74
parents 099b8bcc 25dc7bc3
Pipeline #171312 waiting for manual action with stages
in 20 minutes and 42 seconds
......@@ -36,13 +36,17 @@ class ResourcesController < ApplicationController
raise NotFound, "Cannot find resource #{path}" if object.nil?
if object.has_key?('items')
if params[:deep]
object['links'] = links_for_collection
object['items'].each do |item|
item['links'] = links_for_item(item)
end
else
object['links'] = links_for_item(object)
if object.has_key?('items')
object['links'] = links_for_collection
object['items'].each do |item|
item['links'] = links_for_item(item)
end
else
object['links'] = links_for_item(object)
end
end
object['version'] = repository.commit.oid
......@@ -90,6 +94,10 @@ class ResourcesController < ApplicationController
params[:queues].split(',')
end
end
if params[:controller] == 'sites' && params[:action] == 'show' && params[:deep] && params[:job_id]
params[:version] = OAR::Job.expanded.find(params[:job_id]).start_time
end
end
def lookup_path(path, params)
......@@ -98,7 +106,8 @@ class ResourcesController < ApplicationController
branch: params[:branch],
version: params[:version],
timestamp: params[:timestamp],
date: params[:date]
date: params[:date],
deep: params[:deep]
)
raise ServerUnavailable if object.is_a?(Exception)
......@@ -115,16 +124,40 @@ class ResourcesController < ApplicationController
# 2. case of an array of clusters
when %w[clusters index]
# First, add ["admin","default"] to 'queues' if nothing defined for that cluster
object['items'].each { |cluster| cluster['queues'] = %w[admin default] if cluster['queues'].nil? }
# Then, filter out 'queues' that are not requested in params
object['items'].delete_if { |cluster| (cluster['queues'] & params[:queues]).empty? }
# # This last step: to maintain current behaviour showing no 'queues' if not defined
# # Should be removed when 'queues' in all clusters are explicitly defined.
# object['items'].each { |cluster| cluster.delete_if { |key, value| key == 'queues' && value == ["default"] } }
# Finally, set new 'total' to clusters shortlisted
object['total'] = object['items'].length
unless params[:deep]
# First, add ["admin","default"] to 'queues' if nothing defined for that cluster
object['items'].each { |cluster| cluster['queues'] = %w[admin default] if cluster['queues'].nil? }
# Then, filter out 'queues' that are not requested in params
object['items'].delete_if { |cluster| (cluster['queues'] & params[:queues]).empty? }
# # This last step: to maintain current behaviour showing no 'queues' if not defined
# # Should be removed when 'queues' in all clusters are explicitly defined.
# object['items'].each { |cluster| cluster.delete_if { |key, value| key == 'queues' && value == ["default"] } }
# Finally, set new 'total' to clusters shortlisted
object['total'] = object['items'].length
end
when %w[sites show]
if params[:deep] && params[:job_id]
assigned_nodes = OAR::Job.expanded.find(
params[:job_id]
).assigned_nodes
clusters = {}
assigned_nodes.each do |n|
clusters[n.gsub(/([a-z]+)-[0-9]+.*/, '\1')] ||= []
clusters[n.gsub(/([a-z]+)-[0-9]+.*/, '\1')] << n.gsub(/([a-z]+-[0-9]+).*/, '\1')
end
object['items'].delete_if { |key| !%w[clusters type uid].include?(key) }
object['items']['clusters'].delete_if { |key, _| !clusters.keys.include?(key) }
clusters.each do |cluster, nodes|
if object['items']['clusters'][cluster]
object['items']['clusters'][cluster]['nodes'].delete_if { |key| !nodes.include?(key['uid']) }
end
end
object['total'] = object['items'].length
end
end
object
......@@ -150,32 +183,32 @@ class ResourcesController < ApplicationController
(item.delete('subresources') || []).each do |subresource|
href = uri_to(resource_path(item['uid']) + '/' + subresource[:name])
links.push({
'rel' => subresource[:name],
'href' => href,
'type' => api_media_type(:g5kcollectionjson)
})
'rel' => subresource[:name],
'href' => href,
'type' => api_media_type(:g5kcollectionjson)
})
end
links.push({
'rel' => 'self',
'type' => api_media_type(:g5kitemjson),
'href' => uri_to(resource_path(item['uid']))
})
'rel' => 'self',
'type' => api_media_type(:g5kitemjson),
'href' => uri_to(resource_path(item['uid']))
})
links.push({
'rel' => 'parent',
'type' => api_media_type(:g5kitemjson),
'href' => uri_to(parent_path)
})
'rel' => 'parent',
'type' => api_media_type(:g5kitemjson),
'href' => uri_to(parent_path)
})
links.push({
'rel' => 'version',
'type' => api_media_type(:g5kitemjson),
'href' => uri_to(File.join(resource_path(item['uid']), 'versions', item['version']))
})
'rel' => 'version',
'type' => api_media_type(:g5kitemjson),
'href' => uri_to(File.join(resource_path(item['uid']), 'versions', item['version']))
})
links.push({
'rel' => 'versions',
'type' => api_media_type(:g5kcollectionjson),
'href' => uri_to(File.join(resource_path(item['uid']), 'versions'))
})
'rel' => 'versions',
'type' => api_media_type(:g5kcollectionjson),
'href' => uri_to(File.join(resource_path(item['uid']), 'versions'))
})
links
end
......@@ -183,17 +216,15 @@ class ResourcesController < ApplicationController
def links_for_collection
links = []
links.push({
'rel' => 'self',
'type' => api_media_type(:g5kcollectionjson),
'href' => uri_to(collection_path)
})
unless parent_path.blank?
links.push({
'rel' => 'parent',
'type' => api_media_type(:g5kitemjson),
'href' => uri_to(parent_path)
})
end
'rel' => 'self',
'type' => api_media_type(:g5kcollectionjson),
'href' => uri_to(collection_path)
})
links.push({
'rel' => 'parent',
'type' => api_media_type(:g5kitemjson),
'href' => uri_to(parent_path)
})
links
end
end
......@@ -22,6 +22,7 @@ Api::Application.routes.draw do
get '/versions/:id' => 'versions#show', :via => [:get]
get '*resource/versions' => 'versions#index', :via => [:get]
get '*resource/versions/:id' => 'versions#show', :via => [:get]
get '/sites/:site_id/clusters/:id/nodes' => 'nodes#index', :via => [:get]
resources :environments, only: %i[index show], constraints: { id: /[0-9A-Za-z\-\.]+/ }
resources :network_equipments, only: %i[index show]
......
......@@ -15,6 +15,7 @@
require 'json'
require 'logger'
require 'rugged'
require 'hash'
module Grid5000
class Repository
......@@ -52,7 +53,12 @@ module Grid5000
return e
end
result = expand_object(object, path, @commit)
result = if options[:deep]
deep_expand(object, path, @commit)
else
expand_object(object, path, @commit)
end
result
end
......@@ -110,6 +116,74 @@ module Grid5000
end
end
def deep_expand(hash_object, path, commit)
return nil if hash_object.nil?
tree_object = instance.lookup(hash_object[:oid])
# If it's a symlink
if hash_object[:filemode] == 40_960
hash_object = find_object_at(instance.lookup(entry[:oid]).content, commit, File.join(path, root, entry[:name]))
tree_object = instance.lookup(hash_object[:oid])
end
deep_hash = {}
flat_array = []
sub_hash = {}
last_root = nil
tree_object.walk_blobs(:postorder) do |root, entry|
next unless File.extname(entry[:name]) == '.json'
# If it's a symlink
if entry[:filemode] == 40960
hash_object = find_object_at(instance.lookup(entry[:oid]).content, commit, File.join(path, root, entry[:name]))
object = instance.lookup(hash_object[:oid])
else
object = instance.lookup(entry[:oid])
end
path_hierarchy = File.dirname("#{root}#{entry[:name]}").split('/')
file_hash = JSON.parse(object.content)
last_root = root unless last_root
sub_hash = {} if last_root != root
last_root = root
path_hierarchy = [] if path_hierarchy == ['.']
# If it's a node or a network_equipment, we want to return an Array of
# Hashes
#
# This is also required when we want to return a list (for example a
# list of servers) with no parent. This case happens for a deep view.
if ['nodes', 'network_equipments', 'servers', 'pdus'].include?(path_hierarchy.last) ||
(root.empty? && File.basename(entry[:name], '.json') != path.split('/').last)
if path_hierarchy.empty?
flat_array << file_hash
else
sub_hash[path_hierarchy.last] ||= []
sub_hash[path_hierarchy.last] << file_hash
merge_path_hierarchy = path_hierarchy - [path_hierarchy.last]
deep_hash = deep_hash.deep_merge(Hash.from_array(merge_path_hierarchy, sub_hash))
end
else
file_hash = Hash.from_array(path_hierarchy, file_hash)
deep_hash = deep_hash.deep_merge(file_hash)
end
end
data = deep_hash.empty? ? flat_array : deep_hash
result = {
"total" => data.length,
"offset" => 0,
"items" => rec_sort(data),
"version" => commit.oid
}
result
end
def find_commit_for(options = {})
options[:branch] ||= 'master'
version, branch, timestamp, date = options.values_at(:version, :branch, :timestamp, :date)
......
# Monkey patching Ruby's Hash class
# Extend Hash with helper methods needed to convert input data files to ruby Hash
class Hash
# Add an element composed of nested Hashes made from elements found in "array" argument
# i.e.: from_array([a, b, c],"foo") -> {a: {b: {c: "foo"}}}
def self.from_array(array, value)
return array.reverse.inject(value) { |a, n| { n => a } }
end
end
def rec_sort(h)
case h
when Array
h.map{|v| rec_sort(v)}#.sort_by!{|v| (v.to_s rescue nil) }
when Hash
Hash[Hash[h.map{|k,v| [rec_sort(k),rec_sort(v)]}].sort_by{|k,v| [(k.to_s rescue nil), (v.to_s rescue nil)]}]
else
h
end
end
# Copyright (c) 2009-2011 Cyril Rohr, INRIA Rennes - Bretagne Atlantique
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
require 'spec_helper'
describe NodesController do
render_views
describe "GET /sites/{{site_id}}/clusters/{{cluster_id}}/nodes?deep=true" do
it "should get the correct deep view for one site" do
get :index, params: { site_id: 'rennes', cluster_id: 'paravent', format: :json, deep: true }
expect(response.status).to eq 200
expect(json['total']).to eq 99
expect(json['items'].length).to eq 99
expect(json['items']).to be_a(Array)
expect(json['items'].first['uid']).to eq 'paravent-1'
end
end
end
......@@ -57,4 +57,13 @@ describe RootController do
]
})
end
it "should get the correct deep view" do
get :show, params: { id: 'grid5000', format: :json, deep: true }
expect(response.status).to eq 200
expect(json['total']).to eq 4
expect(json['items'].length).to eq 4
expect(json['items']['sites']).to be_a(Hash)
expect(json['items']['environments']).to be_a(Hash)
end
end
......@@ -37,11 +37,38 @@ describe ServersController do
expect(response.status).to eq(200)
expect(json['total']).to eq(2)
serverList = []
server_list = []
json['items'].each do |server|
serverList = [server['uid']] | serverList
server_list = [server['uid']] | server_list
end
expect(serverList - %w[storage5k talc-data]).to be_empty
expect(server_list - %w[storage5k talc-data]).to be_empty
end
end
describe 'GET /sites/{{site_id}}/servers?deep=true' do
it 'should return 2 servers in site nancy and their exact names' do
get :index, params: { deep: true, site_id: 'nancy', format: :json }
expect(response.status).to eq(200)
expect(json['total']).to eq(2)
expect(json['items']).to be_a(Array)
server_list = []
json['items'].each do |server|
server_list = [server['uid']] | server_list
end
expect(server_list - %w[storage5k talc-data]).to be_empty
end
it 'should return 2 links, parent and self' do
get :index, params: { deep: true, site_id: 'nancy', format: :json }
expect(response.status).to eq(200)
expect(json['links'].length).to eq(2)
links_rel = []
json['links'].each do |link|
links_rel = [link['rel']] | links_rel
end
expect(links_rel - %w[self parent]).to be_empty
end
end
end
......@@ -224,4 +224,47 @@ describe SitesController do
expect(json['nodes']['paramount-4.rennes.grid5000.fr']['reservations']).to_not be_nil
end
end
describe 'GET /sites?deep=true' do
it "should get the correct deep view of sites" do
get :index, params: { format: :json, deep: true }
expect(response.status).to eq 200
expect(json['items'].length).to eq 4
expect(json['items']['bordeaux'].length).to eq 14
expect(json['items']['bordeaux']).to be_a(Hash)
expect(json['items']['bordeaux']['uid']).to eq 'bordeaux'
end
it "should be the correct version" do
get :index, params: { format: :json, deep: true }
expect(response.status).to eq 200
expect(json['version']).to eq '8a562420c9a659256eeaafcfd89dfa917b5fb4d0'
end
end
describe "GET /sites/{{id}}?deep=true" do
it "should get the correct deep view for one site" do
get :show, params: { id: 'rennes', format: :json, deep: true }
expect(response.status).to eq 200
expect(json['total']).to eq 14
expect(json['items'].length).to eq 14
expect(json['items']['clusters']).to be_a(Hash)
expect(json['items']['clusters']['paravent']['uid']).to eq 'paravent'
end
end
describe "GET /sites/{{id}}?deep=true&job_id={{job_id}}" do
it "should get the correct nodes collection for a job" do
get :show, params: { id: 'rennes', job_id: '374191', format: :json, deep: true }
expect(response.status).to eq 200
expect(json['total']).to eq 3
expect(json['items'].length).to eq 3
expect(json['items']['clusters']).to be_a(Hash)
expect(json['items']['clusters']['paramount']['uid']).to eq 'paramount'
expect(json['items']['clusters']['paramount']['nodes']).to be_a(Array)
expect(json['items']['clusters']['paramount']['nodes'].first['uid']).to eq 'paramount-30'
expect(json['items']['clusters']['paramount']['nodes'].length).to eq 4
expect(json['version']).to eq '5b02702daa827f7e39ebf7396af26735c9d2aacd'
end
end
end
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment