Merge branch 'features/#12051_deep' into 'master'

Add the ability to return a deep view of reference-repository

See merge request !74

app/controllers/resources_controller.rb

@@ -36,13 +36,17 @@ class ResourcesController < ApplicationController
 
     raise NotFound, "Cannot find resource #{path}" if object.nil?
 
-    if object.has_key?('items')
+    if params[:deep]
       object['links'] = links_for_collection
-      object['items'].each do |item|
-        item['links'] = links_for_item(item)
-      end
     else
-      object['links'] = links_for_item(object)
+      if object.has_key?('items')
+        object['links'] = links_for_collection
+        object['items'].each do |item|
+          item['links'] = links_for_item(item)
+        end
+      else
+        object['links'] = links_for_item(object)
+      end
     end
 
     object['version'] = repository.commit.oid
@@ -90,6 +94,10 @@ class ResourcesController < ApplicationController
                           params[:queues].split(',')
                                           end
                       end
+
+    if params[:controller] == 'sites' && params[:action] == 'show' && params[:deep] && params[:job_id]
+      params[:version] = OAR::Job.expanded.find(params[:job_id]).start_time
+    end
   end
 
   def lookup_path(path, params)
@@ -98,7 +106,8 @@ class ResourcesController < ApplicationController
       branch: params[:branch],
       version: params[:version],
       timestamp: params[:timestamp],
-      date: params[:date]
+      date: params[:date],
+      deep: params[:deep]
     )
 
     raise ServerUnavailable if object.is_a?(Exception)
@@ -115,16 +124,40 @@ class ResourcesController < ApplicationController
 
     # 2. case of an array of clusters
     when %w[clusters index]
-      # First, add ["admin","default"] to 'queues' if nothing defined for that cluster
-      object['items'].each { |cluster| cluster['queues'] = %w[admin default] if cluster['queues'].nil? }
-      # Then, filter out 'queues' that are not requested in params
-      object['items'].delete_if { |cluster| (cluster['queues'] & params[:queues]).empty? }
-      #          # This last step: to maintain current behaviour showing no 'queues' if not defined
-      #          # Should be removed when 'queues' in all clusters are explicitly defined.
-      #          object['items'].each { |cluster| cluster.delete_if { |key, value| key == 'queues' && value == ["default"] } }
-      # Finally, set new 'total' to clusters shortlisted
-      object['total'] = object['items'].length
+      unless params[:deep]
+        # First, add ["admin","default"] to 'queues' if nothing defined for that cluster
+        object['items'].each { |cluster| cluster['queues'] = %w[admin default] if cluster['queues'].nil? }
+        # Then, filter out 'queues' that are not requested in params
+        object['items'].delete_if { |cluster| (cluster['queues'] & params[:queues]).empty? }
+        #          # This last step: to maintain current behaviour showing no 'queues' if not defined
+        #          # Should be removed when 'queues' in all clusters are explicitly defined.
+        #          object['items'].each { |cluster| cluster.delete_if { |key, value| key == 'queues' && value == ["default"] } }
+        # Finally, set new 'total' to clusters shortlisted
+        object['total'] = object['items'].length
+      end
 
+    when %w[sites show]
+      if params[:deep] && params[:job_id]
+        assigned_nodes = OAR::Job.expanded.find(
+          params[:job_id]
+        ).assigned_nodes
+
+        clusters = {}
+        assigned_nodes.each do |n|
+          clusters[n.gsub(/([a-z]+)-[0-9]+.*/, '\1')] ||= []
+          clusters[n.gsub(/([a-z]+)-[0-9]+.*/, '\1')] << n.gsub(/([a-z]+-[0-9]+).*/, '\1')
+        end
+
+        object['items'].delete_if { |key| !%w[clusters type uid].include?(key) }
+        object['items']['clusters'].delete_if { |key, _| !clusters.keys.include?(key) }
+        clusters.each do |cluster, nodes|
+          if object['items']['clusters'][cluster]
+            object['items']['clusters'][cluster]['nodes'].delete_if { |key| !nodes.include?(key['uid']) }
+          end
+        end
+
+        object['total'] = object['items'].length
+      end
     end
 
     object
@@ -150,32 +183,32 @@ class ResourcesController < ApplicationController
     (item.delete('subresources') || []).each do |subresource|
       href = uri_to(resource_path(item['uid']) + '/' + subresource[:name])
       links.push({
-                   'rel' => subresource[:name],
-                   'href' => href,
-                   'type' => api_media_type(:g5kcollectionjson)
-                 })
+        'rel' => subresource[:name],
+        'href' => href,
+        'type' => api_media_type(:g5kcollectionjson)
+      })
     end
 
     links.push({
-                 'rel' => 'self',
-                 'type' => api_media_type(:g5kitemjson),
-                 'href' => uri_to(resource_path(item['uid']))
-               })
+      'rel' => 'self',
+      'type' => api_media_type(:g5kitemjson),
+      'href' => uri_to(resource_path(item['uid']))
+    })
     links.push({
-                 'rel' => 'parent',
-                 'type' => api_media_type(:g5kitemjson),
-                 'href' => uri_to(parent_path)
-               })
+      'rel' => 'parent',
+      'type' => api_media_type(:g5kitemjson),
+      'href' => uri_to(parent_path)
+    })
     links.push({
-                 'rel' => 'version',
-                 'type' => api_media_type(:g5kitemjson),
-                 'href' => uri_to(File.join(resource_path(item['uid']), 'versions', item['version']))
-               })
+      'rel' => 'version',
+      'type' => api_media_type(:g5kitemjson),
+      'href' => uri_to(File.join(resource_path(item['uid']), 'versions', item['version']))
+    })
     links.push({
-                 'rel' => 'versions',
-                 'type' => api_media_type(:g5kcollectionjson),
-                 'href' => uri_to(File.join(resource_path(item['uid']), 'versions'))
-               })
+      'rel' => 'versions',
+      'type' => api_media_type(:g5kcollectionjson),
+      'href' => uri_to(File.join(resource_path(item['uid']), 'versions'))
+    })
     links
   end
 
@@ -183,17 +216,15 @@ class ResourcesController < ApplicationController
   def links_for_collection
     links = []
     links.push({
-                 'rel' => 'self',
-                 'type' => api_media_type(:g5kcollectionjson),
-                 'href' => uri_to(collection_path)
-               })
-    unless parent_path.blank?
-      links.push({
-                   'rel' => 'parent',
-                   'type' => api_media_type(:g5kitemjson),
-                   'href' => uri_to(parent_path)
-                 })
-    end
+      'rel' => 'self',
+      'type' => api_media_type(:g5kcollectionjson),
+      'href' => uri_to(collection_path)
+    })
+    links.push({
+      'rel' => 'parent',
+      'type' => api_media_type(:g5kitemjson),
+      'href' => uri_to(parent_path)
+    })
     links
   end
 end

config/routes.rb

@@ -22,6 +22,7 @@ Api::Application.routes.draw do
   get '/versions/:id' => 'versions#show', :via => [:get]
   get '*resource/versions' => 'versions#index', :via => [:get]
   get '*resource/versions/:id' => 'versions#show', :via => [:get]
+  get '/sites/:site_id/clusters/:id/nodes' => 'nodes#index', :via => [:get]
 
   resources :environments, only: %i[index show], constraints: { id: /[0-9A-Za-z\-\.]+/	}
   resources :network_equipments, only: %i[index show]

lib/grid5000/repository.rb

@@ -15,6 +15,7 @@
 require 'json'
 require 'logger'
 require 'rugged'
+require 'hash'
 
 module Grid5000
   class Repository
@@ -52,7 +53,12 @@ module Grid5000
         return e
       end
 
-      result = expand_object(object, path, @commit)
+      result = if options[:deep]
+                 deep_expand(object, path, @commit)
+               else
+                 expand_object(object, path, @commit)
+               end
+
       result
     end
 
@@ -110,6 +116,74 @@ module Grid5000
       end
     end
 
+    def deep_expand(hash_object, path, commit)
+      return nil if hash_object.nil?
+
+      tree_object = instance.lookup(hash_object[:oid])
+
+      # If it's a symlink
+      if hash_object[:filemode] == 40_960
+        hash_object = find_object_at(instance.lookup(entry[:oid]).content, commit, File.join(path, root, entry[:name]))
+        tree_object = instance.lookup(hash_object[:oid])
+      end
+
+      deep_hash = {}
+      flat_array = []
+      sub_hash = {}
+      last_root = nil
+      tree_object.walk_blobs(:postorder) do |root, entry|
+        next unless File.extname(entry[:name]) == '.json'
+
+        # If it's a symlink
+        if entry[:filemode] == 40960
+          hash_object = find_object_at(instance.lookup(entry[:oid]).content, commit, File.join(path, root, entry[:name]))
+          object = instance.lookup(hash_object[:oid])
+        else
+          object = instance.lookup(entry[:oid])
+        end
+
+        path_hierarchy = File.dirname("#{root}#{entry[:name]}").split('/')
+        file_hash = JSON.parse(object.content)
+
+        last_root = root unless last_root
+        sub_hash = {} if last_root != root
+        last_root = root
+
+        path_hierarchy = [] if path_hierarchy == ['.']
+
+        # If it's a node or a network_equipment, we want to return an Array of
+        # Hashes
+        #
+        # This is also required when we want to return a list (for example a
+        # list of servers) with no parent. This case happens for a deep view.
+        if ['nodes', 'network_equipments', 'servers', 'pdus'].include?(path_hierarchy.last) ||
+            (root.empty? && File.basename(entry[:name], '.json') != path.split('/').last)
+
+          if path_hierarchy.empty?
+            flat_array << file_hash
+          else
+            sub_hash[path_hierarchy.last] ||= []
+            sub_hash[path_hierarchy.last] << file_hash
+            merge_path_hierarchy = path_hierarchy - [path_hierarchy.last]
+            deep_hash = deep_hash.deep_merge(Hash.from_array(merge_path_hierarchy, sub_hash))
+          end
+        else
+          file_hash = Hash.from_array(path_hierarchy, file_hash)
+          deep_hash = deep_hash.deep_merge(file_hash)
+        end
+      end
+
+      data = deep_hash.empty? ? flat_array : deep_hash
+
+      result = {
+        "total" => data.length,
+        "offset" => 0,
+        "items" => rec_sort(data),
+        "version" => commit.oid
+      }
+      result
+    end
+
     def find_commit_for(options = {})
       options[:branch] ||= 'master'
       version, branch, timestamp, date = options.values_at(:version, :branch, :timestamp, :date)

lib/hash.rb

+# Monkey patching Ruby's Hash class
+
+# Extend Hash with helper methods needed to convert input data files to ruby Hash
+class Hash
+  # Add an element composed of nested Hashes made from elements found in "array" argument
+  # i.e.: from_array([a, b, c],"foo") -> {a: {b: {c: "foo"}}}
+  def self.from_array(array, value)
+    return array.reverse.inject(value) { |a, n| { n => a } }
+  end
+end
+
+def rec_sort(h)
+  case h
+  when Array
+    h.map{|v| rec_sort(v)}#.sort_by!{|v| (v.to_s rescue nil) }
+  when Hash
+    Hash[Hash[h.map{|k,v| [rec_sort(k),rec_sort(v)]}].sort_by{|k,v| [(k.to_s rescue nil), (v.to_s rescue nil)]}]
+  else
+    h
+  end
+end

spec/controllers/nodes_controller_spec.rb

+# Copyright (c) 2009-2011 Cyril Rohr, INRIA Rennes - Bretagne Atlantique
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+require 'spec_helper'
+
+describe NodesController do
+  render_views
+
+  describe "GET /sites/{{site_id}}/clusters/{{cluster_id}}/nodes?deep=true" do
+    it "should get the correct deep view for one site" do
+      get :index, params: { site_id: 'rennes', cluster_id: 'paravent', format: :json, deep: true }
+      expect(response.status).to eq 200
+      expect(json['total']).to eq 99
+      expect(json['items'].length).to eq 99
+      expect(json['items']).to be_a(Array)
+      expect(json['items'].first['uid']).to eq 'paravent-1'
+    end
+  end
+end

spec/controllers/root_controller_spec.rb

@@ -57,4 +57,13 @@ describe RootController do
                          ]
                        })
   end
+
+  it "should get the correct deep view" do
+    get :show, params: { id: 'grid5000', format: :json, deep: true }
+    expect(response.status).to eq 200
+    expect(json['total']).to eq 4
+    expect(json['items'].length).to eq 4
+    expect(json['items']['sites']).to be_a(Hash)
+    expect(json['items']['environments']).to be_a(Hash)
+  end
 end

spec/controllers/servers_controller_spec.rb

@@ -37,11 +37,38 @@ describe ServersController do
       expect(response.status).to eq(200)
       expect(json['total']).to eq(2)
 
-      serverList = []
+      server_list = []
       json['items'].each do |server|
-        serverList = [server['uid']] | serverList
+        server_list = [server['uid']] | server_list
       end
-      expect(serverList - %w[storage5k talc-data]).to be_empty
+      expect(server_list - %w[storage5k talc-data]).to be_empty
+    end
+  end
+
+  describe 'GET /sites/{{site_id}}/servers?deep=true' do
+    it 'should return 2 servers in site nancy and their exact names' do
+      get :index, params: { deep: true, site_id: 'nancy', format: :json }
+      expect(response.status).to eq(200)
+      expect(json['total']).to eq(2)
+      expect(json['items']).to be_a(Array)
+
+      server_list = []
+      json['items'].each do |server|
+        server_list = [server['uid']] | server_list
+      end
+      expect(server_list - %w[storage5k talc-data]).to be_empty
+    end
+
+    it 'should return 2 links, parent and self' do
+      get :index, params: { deep: true, site_id: 'nancy', format: :json }
+      expect(response.status).to eq(200)
+      expect(json['links'].length).to eq(2)
+
+      links_rel = []
+      json['links'].each do |link|
+        links_rel = [link['rel']] | links_rel
+      end
+      expect(links_rel - %w[self parent]).to be_empty
     end
   end
 end

spec/controllers/sites_controller_spec.rb

@@ -224,4 +224,47 @@ describe SitesController do
       expect(json['nodes']['paramount-4.rennes.grid5000.fr']['reservations']).to_not be_nil
     end
   end
+
+  describe 'GET /sites?deep=true' do
+    it "should get the correct deep view of sites" do
+      get :index, params: { format: :json, deep: true }
+      expect(response.status).to eq 200
+      expect(json['items'].length).to eq 4
+      expect(json['items']['bordeaux'].length).to eq 14
+      expect(json['items']['bordeaux']).to be_a(Hash)
+      expect(json['items']['bordeaux']['uid']).to eq 'bordeaux'
+    end
+
+    it "should be the correct version" do
+      get :index, params: { format: :json, deep: true }
+      expect(response.status).to eq 200
+      expect(json['version']).to eq '8a562420c9a659256eeaafcfd89dfa917b5fb4d0'
+    end
+  end
+
+  describe "GET /sites/{{id}}?deep=true" do
+    it "should get the correct deep view for one site" do
+      get :show, params: { id: 'rennes', format: :json, deep: true }
+      expect(response.status).to eq 200
+      expect(json['total']).to eq 14
+      expect(json['items'].length).to eq 14
+      expect(json['items']['clusters']).to be_a(Hash)
+      expect(json['items']['clusters']['paravent']['uid']).to eq 'paravent'
+    end
+  end
+
+  describe "GET /sites/{{id}}?deep=true&job_id={{job_id}}" do
+    it "should get the correct nodes collection for a job" do
+      get :show, params: { id: 'rennes', job_id: '374191', format: :json, deep: true }
+      expect(response.status).to eq 200
+      expect(json['total']).to eq 3
+      expect(json['items'].length).to eq 3
+      expect(json['items']['clusters']).to be_a(Hash)
+      expect(json['items']['clusters']['paramount']['uid']).to eq 'paramount'
+      expect(json['items']['clusters']['paramount']['nodes']).to be_a(Array)
+      expect(json['items']['clusters']['paramount']['nodes'].first['uid']).to eq 'paramount-30'
+      expect(json['items']['clusters']['paramount']['nodes'].length).to eq 4
+      expect(json['version']).to eq '5b02702daa827f7e39ebf7396af26735c9d2aacd'
+    end
+  end
 end