Commit ce5a8871 authored by Samir Noir's avatar Samir Noir 🧀
Browse files

Replace Grit by Rugged libgit2 binding for accessing git repository

parent db0a2d0d
Pipeline #156002 passed with stages
in 19 minutes and 12 seconds
......@@ -12,12 +12,12 @@ gem "ruby-mysql", :require => "mysql"
gem 'addressable', '~> 2.2'
gem 'thin', '~> 1.7.0'
gem 'state_machines-activerecord'
gem 'gitlab-grit', :require => ['grit']
gem 'syslogger'
gem 'haml', '~> 5.1.2'
gem 'rack-jsonp'
gem 'pg'
gem 'nokogiri'
gem 'rugged'
gem 'sass-rails'
gem 'coffee-rails'
......
......@@ -62,7 +62,6 @@ GEM
msgpack (~> 1.0)
builder (3.2.4)
byebug (11.1.3)
charlock_holmes (0.7.7)
coffee-rails (5.0.0)
coffee-script (>= 2.2.0)
railties (>= 5.2.0)
......@@ -89,11 +88,6 @@ GEM
factory_bot (~> 5.2.0)
railties (>= 4.2.0)
ffi (1.12.2)
gitlab-grit (2.8.3)
charlock_holmes (~> 0.7)
diff-lcs (~> 1.1)
mime-types (>= 1.16, < 3)
posix-spawn (~> 0.3)
globalid (0.4.2)
activesupport (>= 4.2.0)
haml (5.1.2)
......@@ -137,7 +131,6 @@ GEM
nokogiri (1.10.9)
mini_portile2 (~> 2.4.0)
pg (1.2.3)
posix-spawn (0.3.13)
public_suffix (4.0.5)
rack (2.2.2)
rack-fiber_pool (0.9.3)
......@@ -206,6 +199,7 @@ GEM
net-ssh-multi (>= 1.2)
rest-client (>= 1.6)
ruby-mysql (2.9.14)
rugged (1.0.1)
safe_yaml (1.0.5)
sass-rails (6.0.0)
sassc-rails (~> 2.1, >= 2.1.1)
......@@ -270,7 +264,6 @@ DEPENDENCIES
coffee-rails
erubis (~> 2.7)
factory_bot_rails
gitlab-grit
haml (~> 5.1.2)
jquery-rails
mysql2 (~> 0.5.3)
......@@ -287,6 +280,7 @@ DEPENDENCIES
rspec_junit_formatter (~> 0.3.0)
ruby-cute
ruby-mysql
rugged
sass-rails
simplecov
state_machines-activerecord
......
......@@ -45,9 +45,9 @@ class ResourcesController < ApplicationController
object['links'] = links_for_item(object)
end
object["version"] = repository.commit.id
object["version"] = repository.commit.oid
last_modified [repository.commit.committed_date, File.mtime(__FILE__)].max
last_modified [repository.commit.time, File.mtime(__FILE__)].max
# If client asked for a specific version, it won't change anytime soon
if params[:version] && params[:version] == object["version"]
......@@ -152,9 +152,9 @@ class ResourcesController < ApplicationController
links = []
(item.delete('subresources') || []).each do |subresource|
href = uri_to(resource_path(item["uid"]) + "/" + subresource.name)
href = uri_to(resource_path(item["uid"]) + "/" + subresource[:name])
links.push({
"rel" => subresource.name,
"rel" => subresource[:name],
"href" => href,
"type" => api_media_type(:g5kcollectionjson)
})
......
......@@ -14,92 +14,92 @@
class VersionsController < ApplicationController
MAX_AGE = 60.seconds
def index
vary_on :accept; allow :get
versions = repository.versions_for(
resource_path,
:branch => params[:branch],
:offset => params[:offset],
resource_path,
:branch => params[:branch],
:offset => params[:offset],
:limit => params[:limit]
)
raise NotFound, "#{resource_path} does not exist." if versions["total"] == 0
versions["items"].map!{|commit|
metadata_for_commit(commit, resource_path)
}
versions["links"] = [
{
"rel" => "self",
"href" => uri_to("#{resource_path}/versions"),
"rel" => "self",
"href" => uri_to("#{resource_path}/versions"),
"type" => api_media_type(:g5kcollectionjson)
},
{
"rel" => "parent",
"href" => uri_to("#{resource_path.split("/")[0..-2].join("/")}"),
"rel" => "parent",
"href" => uri_to("#{resource_path.split("/")[0..-2].join("/")}"),
"type" => api_media_type(:g5kitemjson)
}
]
etag versions.hash
expires_in MAX_AGE, :public => true
respond_to do |format|
format.g5kcollectionjson { render :json => versions }
format.json { render :json => versions }
end
end
def show
vary_on :accept; allow :get
version = params[:id]
versions = repository.versions_for(
resource_path,
:branch => version,
:offset => 0,
resource_path,
:branch => version,
:offset => 0,
:limit => 1
)
raise NotFound, "The requested version '#{version}' does not exist or the resource '#{resource_path}' does not exist." if versions["total"] == 0
# etag compute_etag(commit.id, resource_uri, response['Content-Type'], options.release_hash)
output = metadata_for_commit(versions["items"][0], resource_path)
etag versions.hash
expires_in MAX_AGE, :public => true
respond_to do |format|
format.g5kitemjson { render :json => output }
format.json { render :json => output }
end
end
protected
def resource_path
@resource_path ||= params[:resource].gsub(/\/?platforms/, '')
end
def metadata_for_commit(commit, resource_path)
{
'uid' => commit.id,
'date' => commit.committed_date.httpdate,
{
'uid' => commit.oid,
'date' => commit.time.httpdate,
'message' => commit.message,
'author' => commit.author.name,
'author' => commit.author[:name],
'type' => 'version',
'links' => [
{
"rel" => "self",
"href" => uri_to("#{resource_path}/versions/#{commit.id}"),
"rel" => "self",
"href" => uri_to("#{resource_path}/versions/#{commit.oid}"),
"type" => api_media_type(:g5kitemjson)
},
{
"rel" => "parent",
"href" => uri_to(resource_path),
"rel" => "parent",
"href" => uri_to(resource_path),
"type" => api_media_type(:g5kitemjson)
}
]
]
}
end
end
......@@ -3,7 +3,7 @@ Section: ruby
Priority: extra
Maintainer: Grid'5000 developpers <g5k-developers@lists.grid5000.fr>
Uploaders: David Margery <david.margery@inria.fr>
Build-Depends: debhelper (>= 7), dh-systemd (>= 1.5), git, bundler, rake, libssl-dev, libxml2-dev, libxslt-dev, libicu-dev, default-libmysqlclient-dev | libmysqlclient-dev, libpq-dev, nodejs, ruby-dev, lsb-release, postgresql-client, pkg-config
Build-Depends: debhelper (>= 7), dh-systemd (>= 1.5), git, bundler, rake, libssl-dev, libxml2-dev, libxslt-dev, libicu-dev, default-libmysqlclient-dev | libmysqlclient-dev, libpq-dev, nodejs, ruby-dev, lsb-release, postgresql-client, pkg-config, cmake
Standards-Version: 3.7.3
Homepage: https://api.grid5000.fr
......
# Copyright (c) 2009-2011 Cyril Rohr, INRIA Rennes - Bretagne Atlantique
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
require 'grit'
module Grit
module GitRuby
class Repository
# Added support for :until
def rev_list(sha, options)
if sha.is_a? Array
(end_sha, sha) = sha
end
log = log(sha, options)
log = log.sort { |a, b| a[2] <=> b[2] }.reverse
if end_sha
log = truncate_arr(log, end_sha)
end
if options[:until]
limit = Time.parse(options[:until])
while log.length > 0
if log[0][2] > limit
log.shift
else
break
end
end
end
# shorten the list if it's longer than max_count (had to get everything in branches)
if options[:max_count]
if (opt_len = options[:max_count].to_i) < log.size
log = log[0, opt_len]
end
end
if options[:pretty] == 'raw'
log.map {|k, v| v }.join('')
else
log.map {|k, v| k }.join("\n")
end
end
end
end
end
......@@ -12,9 +12,9 @@
# See the License for the specific language governing permissions and
# limitations under the License.
require 'grid5000/extensions/grit'
require 'json'
require 'logger'
require 'rugged'
module Grid5000
class Repository
......@@ -31,7 +31,7 @@ module Grid5000
@logger = Logger.new(STDOUT)
@logger.level = Logger::WARN
end
@instance = Grit::Repo.new(repository_path)
@instance = Rugged::Repository.new(repository_path)
end
def find(path, options = {})
......@@ -41,15 +41,16 @@ module Grid5000
@commit = nil
begin
@commit = find_commit_for(options)
logger.info " commit = #{@commit.inspect}"
logger.info " commit = #{@commit} {id: #{@commit.oid}, message: #{@commit.message.chomp}}"
return nil if @commit.nil?
object = find_object_at(path, @commit)
logger.debug " object = #{object.inspect}"
logger.debug " object = #{object}"
return nil if object.nil?
rescue Grit::Git::GitTimeout => e
logger.debug "#{Time.now}: Got a Grit::Git::GitTimeout exception #{e}"
rescue => e
logger.debug "#{Time.now}: Got a Rugged exception #{e}"
return e
end
result = expand_object(object, path, @commit)
result
end
......@@ -58,36 +59,41 @@ module Grid5000
File.join(repository_path_prefix, path)
end
def expand_object(object, path, commit)
return nil if object.nil?
def expand_object(hash_object, path, commit)
return nil if hash_object.nil?
object = instance.lookup(hash_object[:oid])
if object.mode == "120000"
object = find_object_at(object.data, commit, relative_to=path)
# If it's a symlink
if hash_object[:filemode] == 40960
hash_object = find_object_at(object.content, commit, relative_to=path)
object = instance.lookup(hash_object[:oid])
end
case object
when Grit::Blob
case object.type
when :blob
@subresources = []
JSON.parse(object.data).merge("version" => commit.id)
when Grit::Tree
groups = object.contents.group_by{|content| content.class}
blobs, trees = [groups[Grit::Blob] || [], groups[Grit::Tree] || []]
JSON.parse(object.content).merge("version" => commit.oid)
when :tree
groups = object.each.group_by{|content| content[:type]}
blobs, trees = [groups[:blob] || [], groups[:tree] || []]
# select only json files
blobs = blobs.select{|blob| File.extname(blob.name) == '.json'}
blobs = blobs.select{|blob| File.extname(blob[:name]) == '.json'}
if (blobs.size > 0 && trees.size > 0) # item
blobs.inject({'subresources' => trees}) do |accu, blob|
content = expand_object(
blob,
File.join(path, blob.name.gsub(".json", "")),
File.join(path, blob[:name].gsub(".json", "")),
commit
)
accu.merge(content)
end
else # collection
items = object.contents.map do |object|
items = object.map do |object|
content = expand_object(
object,
File.join(path, object.name.gsub(".json", "")),
File.join(path, object[:name].gsub(".json", "")),
commit
)
end
......@@ -95,7 +101,7 @@ module Grid5000
"total" => items.length,
"offset" => 0,
"items" => items,
"version" => commit.id
"version" => commit.oid
}
result
end
......@@ -108,25 +114,43 @@ module Grid5000
options[:branch] ||= 'master'
version, branch = options.values_at(:version, :branch)
if version.nil?
instance.commits(branch)[0]
instance.branches[branch].target
elsif version.to_s.length == 40 # SHA
instance.commit(version)
instance.lookup(version)
else
# version should be a date, get the closest commit
date = Time.at(version.to_i).strftime("%Y-%m-%d %H:%M:%S")
sha = instance.git.rev_list({
:pretty => :raw, :until => date
}, branch)
sha = sha.split("\n")[0]
date = version.to_i
return nil if instance.branches[branch].nil?
walker = Rugged::Walker.new(instance)
walker.sorting(Rugged::SORT_DATE)
walker.push(instance.branches[branch].target.oid)
commits = walker.select do |commit|
commit.epoch_time <= date
end
commits.map! { |commit| commit.oid }
sha = commits.first
find_commit_for(options.merge(:version => sha))
end
rescue Grit::GitRuby::Repository::NoSuchShaFound => e
rescue Rugged::OdbError
nil
end
def find_object_at(path, commit, relative_to = nil)
path = relative_path_to(path, relative_to) unless relative_to.nil?
object = commit.tree/path || commit.tree/(path+".json")
begin
object = commit.tree.path(path)
rescue Rugged::TreeError
begin
object = commit.tree.path(path + '.json')
rescue Rugged::TreeError
nil
end
end
end
# Return the physical path within the repository
......@@ -149,21 +173,44 @@ module Grid5000
offset = (offset || 0).to_i
limit = (limit || 100).to_i
path = full_path(path)
commits = instance.log(
branch,
path
)
commits = instance.log(
branch,
path+".json"
) if commits.empty?
commits = []
if instance.branches.exist?(branch)
oid = instance.branches[branch].target.oid
else
begin
if instance.exists?(branch)
oid = instance.lookup(branch).oid
else
oid = nil
end
rescue
oid = nil
end
end
if oid
walker = Rugged::Walker.new(instance)
walker.sorting(Rugged::SORT_DATE)
walker.push(oid)
commits = walker.select do |commit|
commit.diff(paths: [path]).size > 0
end
if commits.empty?
commits = walker.select do |commit|
commit.diff(paths: [path + '.json']).size > 0
end
end
end
{
"total" => commits.length,
"offset" => offset,
"items" => commits.slice(offset, limit)
}
end
end
end # module Grid5000
......@@ -199,7 +199,7 @@ describe SitesController do
end
it "should fail gracefully in the event of a grit timeout" do
expect_any_instance_of(Grid5000::Repository).to receive(:find_commit_for).and_raise(Grit::Git::GitTimeout)
expect_any_instance_of(Grid5000::Repository).to receive(:find_commit_for).and_raise(Rugged::RepositoryError)
get :status, params: { :id => "rennes", :job_details => "no", :format => :json }
expect(response.status).to eq 503
end
......
......@@ -34,7 +34,7 @@ describe Grid5000::Repository do
"/does/not/exist",
@repository_path_prefix
)
}).to raise_error(Grit::NoSuchPathError)
}).to raise_error(Rugged::OSError)
end
describe "with a working repository" do
......@@ -52,35 +52,36 @@ describe Grid5000::Repository do
expect(object['version']).to eq @latest_commit
end
it "should return an exception if Grit::Git::GitTimeout is raised" do
expect(@repository).to receive(:find_commit_for).and_raise(Grit::Git::GitTimeout)
it "should return an exception if Rugged::PathError is raised" do
expect(@repository).to receive(:find_commit_for).and_raise(Rugged::RepositoryError)
object=@repository.find('grid5000')
expect(object).to be_an(Exception)
end
end
describe "finding a specific version" do
it "should return the latest commit of master if no specific version is given" do
commit = @repository.find_commit_for(:version => nil)
expect(commit.id).to eq(@latest_commit)
expect(commit.oid).to eq(@latest_commit)
end
it "should find the commit associated with the given version [version=DATE] 1/2" do
date = Time.parse("2009-03-13 17:24:20 +0100")
commit = @repository.find_commit_for(:version => date.to_i)
expect(commit.id).to eq("b00bd30bf69c322ffe9aca7a9f6e3be0f29e20f4")
expect(commit.oid).to eq("b00bd30bf69c322ffe9aca7a9f6e3be0f29e20f4")
end
it "should find the commit associated with the given version [version=DATE] 2/2" do
date = Time.parse("2009-03-13 17:24:47 +0100")
commit = @repository.find_commit_for(:version => date.to_i)
expect(commit.id).to eq("e07895a4b480aaa8e11c35549a97796dcc4a307d")
expect(commit.oid).to eq("e07895a4b480aaa8e11c35549a97796dcc4a307d")
end
it "should find the commit associated with the given version [version=SHA]" do
commit = @repository.find_commit_for(
:version => "e07895a4b480aaa8e11c35549a97796dcc4a307d"
)
expect(commit.id).to eq("e07895a4b480aaa8e11c35549a97796dcc4a307d")
expect(commit.oid).to eq("e07895a4b480aaa8e11c35549a97796dcc4a307d")
end
it "should return nil when asking for a version from a branch that does not exist" do
......@@ -108,32 +109,38 @@ describe Grid5000::Repository do
end
it "should find a tree object" do
object = @repository.find_object_at(
hash_object = @repository.find_object_at(
@repository.full_path('grid5000'), @commit)
expect(object).to be_a(Grit::Tree)
object = @repository.instance.lookup(hash_object[:oid])
expect(object).to be_a(Rugged::Tree)
end
it "should find a relative object (symlink)" do
relative_to=@repository.full_path(
'grid5000/sites/rennes/environments/sid-x64-base-1.0.json'
)
object = @repository.find_object_at(
hash_object = @repository.find_object_at(
'../../../../grid5000/environments/sid-x64-base-1.0.json',
@commit,
relative_to)
expect(object).to be_a(Grit::Blob)
expect(object.data).to match /kernel/
object = @repository.instance.lookup(hash_object[:oid])
expect(object).to be_a(Rugged::Blob)
expect(object.content).to match /kernel/
end
it "should find a blob" do
object = @repository.find_object_at(
hash_object = @repository.find_object_at(
@repository.full_path(
'grid5000/environments/sid-x64-base-1.0.json'
),
@commit
)
expect(object).to be_a(Grit::Blob)
expect(object.data).to match /kernel/
object = @repository.instance.lookup(hash_object[:oid])
expect(object).to be_a(Rugged::Blob)
expect(object.content).to match /kernel/
end