Mentions légales du service
Skip to content
GitLab
Explore
Sign in
Primary navigation
Search or go to…
Project
python_client
Manage
Activity
Members
Labels
Plan
Issues
Issue boards
Milestones
Wiki
Code
Merge requests
Repository
Branches
Commits
Tags
Repository graph
Compare revisions
Snippets
Build
Pipelines
Jobs
Pipeline schedules
Artifacts
Deploy
Releases
Model registry
Operate
Environments
Monitor
Incidents
Service Desk
Analyze
Value stream analytics
Contributor analytics
CI/CD analytics
Repository analytics
Model experiments
Help
Help
Support
GitLab documentation
Compare GitLab plans
Community forum
Contribute to GitLab
Provide feedback
Keyboard shortcuts
?
Snippets
Groups
Projects
Show more breadcrumbs
allgo
API-Clients
python_client
Commits
63272da1
Commit
63272da1
authored
5 years ago
by
LETORT Sebastien
Browse files
Options
Downloads
Patches
Plain Diff
New method to download files, in python2 and python3.
parent
e7a6810e
Branches
Branches containing commit
Tags
Tags containing commit
1 merge request
!4
Client api
Changes
3
Hide whitespace changes
Inline
Side-by-side
Showing
3 changed files
allgo/__init__.py
+109
-0
109 additions, 0 deletions
allgo/__init__.py
notebooks/demo_short.ipynb
+49
-0
49 additions, 0 deletions
notebooks/demo_short.ipynb
tests/test_allgo.py
+88
-0
88 additions, 0 deletions
tests/test_allgo.py
with
246 additions
and
0 deletions
allgo/__init__.py
+
109
−
0
View file @
63272da1
...
...
@@ -45,6 +45,7 @@ and finally :
# standard libs
from
__future__
import
print_function
import
errno
import
logging
import
os
import
sys
...
...
@@ -346,6 +347,114 @@ class Client:
# --------------------------------------------
# -- Accessory methods, to ease user work.
# --------------------------------------------
def
_request_file
(
self
,
file_url
):
"""
Build and launch a http request on any URL, but with Allgo authentication.
The file downloads as part of the API, it is an additionnal service.
Parameters
----------
file_url : string/URL
the URL to download the file from.
Returns
-------
filepath
the local filepath of the file downloaded.
Raises
------
StatusError
with the Response status code if it
'
s not 200.
"""
headers
=
{
'
Authorization
'
:
'
Token token={}
'
.
format
(
self
.
token
)}
resp
=
requests
.
get
(
file_url
,
headers
=
headers
,
verify
=
self
.
verify_tls
)
if
resp
.
status_code
!=
requests
.
codes
.
ok
:
raise
StatusError
(
resp
.
status_code
,
resp
)
return
resp
def
download_file
(
self
,
*
args
,
**
kwargs
):
"""
download the file pointed by the url.
The file is downloaded in the outdir directory (which is created if not exist).
The old Allgo site (allgo.inria.fr) doesn
'
t need authentication.
The new allgo site (allgo18.inria.fr) does need authentication.
Parameters
----------
file_url : string/URL
the URL to download the file from.
outdir : dirpath, optional
the dirpath to write file to.
force : bool, optional
set to true to force erasing existing file.
Returns
-------
filepath
the local filepath of the file downloaded.
Raises
------
FileExistsError
if the output file exists and force is false.
StatusError
with the Response status code if it
'
s not 200.
"""
if
2
==
sys
.
version_info
.
major
:
return
self
.
__download_file_p2
(
*
args
,
**
kwargs
)
return
self
.
__download_file_p3
(
*
args
,
**
kwargs
)
def
__download_file_p3
(
self
,
file_url
,
outdir
=
'
.
'
,
force
=
False
):
mode
=
'
wb
'
if
force
else
'
xb
'
resp
=
self
.
_request_file
(
file_url
)
# urlparse is better, but a pain to use in python2 & 3
# fileurl, if come from job_status doesn't have params, so it's ok.
filename
=
os
.
path
.
basename
(
file_url
)
os
.
makedirs
(
outdir
,
exist_ok
=
True
)
outfilepath
=
os
.
path
.
join
(
outdir
,
filename
)
with
open
(
outfilepath
,
mode
)
as
file_out
:
# Note: '8192' = 2^13 : size of cache/buffer.
# = max size transfered in one operation.
for
chunk
in
resp
.
iter_content
(
chunk_size
=
8192
):
if
chunk
:
file_out
.
write
(
chunk
)
return
outfilepath
def
__download_file_p2
(
self
,
file_url
,
outdir
=
'
.
'
,
force
=
False
):
# python2: outdir cannot be a PosixPath (cf makedirs, os.path.join)
outdir
=
str
(
outdir
)
filename
=
os
.
path
.
basename
(
file_url
)
outfilepath
=
os
.
path
.
join
(
outdir
,
filename
)
# method with os.fdopen(os.open(outfilepath, mode), 'w') doesn't exist
if
os
.
path
.
isfile
(
outfilepath
)
and
not
force
:
msg
=
"
file {} exists.
"
raise
OSError
(
errno
.
EEXIST
,
msg
)
try
:
os
.
makedirs
(
outdir
)
except
OSError
as
e
:
# silence directory exists error
if
e
.
errno
!=
errno
.
EEXIST
:
raise
resp
=
self
.
_request_file
(
file_url
)
with
open
(
outfilepath
,
'
wb
'
)
as
file_out
:
# Note: '8192' = 2^13 : size of cache/buffer.
# = max size transfered in one operation.
for
chunk
in
resp
.
iter_content
(
chunk_size
=
8192
):
if
chunk
:
file_out
.
write
(
chunk
)
return
outfilepath
def
run_job
(
self
,
app
,
version
=
None
,
params
=
''
,
files
=
None
,
sleep_duration
=
2
,
verbose
=
False
):
"""
Create a job and wait for it to terminate.
...
...
This diff is collapsed.
Click to expand it.
notebooks/demo_short.ipynb
+
49
−
0
View file @
63272da1
...
...
@@ -200,6 +200,55 @@
"print(out_dict)"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"### download a file generated"
]
},
{
"cell_type": "code",
"execution_count": 10,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"We will download 'allgo.log' from job '787' in 'output_dir'.\n",
"downloaded file : output_dir/allgo.log\n",
"...\t\n",
"...\tThis is app 'sleep' called with parameters '3'\n",
"...\t\n",
"...\tThe workdir contains:\n",
"...\t\n",
"...\ttotal 0\n",
"...\t-rw-r--r-- 1 500 500 72 Dec 6 08:49 allgo.log\n",
"...\tPOUET v1\n",
"...\t+ sleep 3\n",
"...\t\n",
"...\t==== ALLGO JOB SUCCESS ====\n"
]
}
],
"source": [
"# last download a file generated\n",
"# here the log file.\n",
"wanted_filename = 'allgo.log'\n",
"outdir = \"output_dir\"\n",
"print(\"We will download '{}' from job '{}' in '{}'.\".format(wanted_filename, job_id, outdir))\n",
"\n",
"job_id = out_dict['id']\n",
"url = out_dict[str(job_id)]['allgo.log']\n",
"filepath = client.download_file(url, force=True, outdir=outdir)\n",
"print(\"downloaded file : {}\".format(filepath))\n",
"\n",
"with open(filepath, 'r') as fe:\n",
" for line in fe:\n",
" print(\"...\\t{}\".format(line), end='')"
]
},
{
"cell_type": "markdown",
"metadata": {},
...
...
%% Cell type:markdown id: tags:
# Allgo Python API-client demonstrator
## Introduction
### reminder
*
Allgo project : https://allgo18.inria.fr
*
Allgo API documentation legacy : https://allgo.inria.fr/documentation/api
*
Allgo API documentation : https://allgo.gitlabpages.inria.fr/doc/api.html
*
API-client project : https://gitlab.inria.fr/allgo/api-clients
### this is a python demonstrator
Here we will show you how to use the python client to launch allgo-job.
## CODE :)
### import allgo
%% Cell type:code id: tags:
```
python
# first import allgo
import
allgo
# you can check
print
(
allgo
.
__version__
)
# print(allgo.__file__)
```
%% Output
0.2.0
%% Cell type:markdown id: tags:
### constructor - an allgo object represents the user-instance connexion
%% Cell type:code id: tags:
```
python
# you could specify an alternative url
token
=
None
# put your token.
url
=
"
https://localhost/
"
# here this is our dev instance.
client
=
allgo
.
Client
(
token
,
allgo_url
=
url
,
verify_tls
=
False
)
# on localhost we cannot verify tls.
print
(
"
You
'
ve got a connexion to {}
"
.
format
(
client
.
allgo_url
))
```
%% Output
You've got a connexion to https://localhost/
%% Cell type:code id: tags:
```
python
# by default, it's the "official" instance that is used.
token
=
None
# put your token.
client
=
allgo
.
Client
(
token
)
print
(
"
You
'
ve got a connexion to {}
"
.
format
(
client
.
allgo_url
))
```
%% Output
You've got a connexion to https://allgo18.inria.fr
%% Cell type:markdown id: tags:
### create a job
%% Cell type:code id: tags:
```
python
app
=
'
sleep
'
# the appli you want to execute
params
=
'
3
'
# sleep duration
files
=
None
# sleep do not use file, else create a list of filepath to load.
# your code allgo
out_dict
=
client
.
create_job
(
app
,
params
=
params
,
files
=
files
)
job_id
=
out_dict
[
'
id
'
]
print
(
"
=> Jobs launched with Id {}.
"
.
format
(
job_id
))
print
(
"
full output dictionary : {}
"
.
format
(
out_dict
))
```
%% Output
=> Jobs launched with Id 787.
full output dictionary : {'url': 'https://allgo18.inria.fr/api/v1/jobs/787', 'avg_time': 0, 'id': 787}
%% Cell type:markdown id: tags:
### get job status
%% Cell type:code id: tags:
```
python
# wait for the job to terminate
# = sleep until its status tell it's terminated.
from
time
import
sleep
while
True
:
out_dict
=
client
.
job_status
(
job_id
)
print
(
"
out_dict = {}
"
.
format
(
out_dict
))
if
out_dict
[
'
status
'
]
!=
'
done
'
:
print
(
"
... not done, let
'
s wait 2 more sec.
"
)
sleep
(
2
)
else
:
print
(
"
=> job
'
s done !
"
)
break
```
%% Output
out_dict = {'787': {'allgo.log': 'https://allgo18.inria.fr/datastore/787/allgo.log'}, 'status': 'done'}
=> job's done !
%% Cell type:markdown id: tags:
### trick to run a job in a single call
%% Cell type:code id: tags:
```
python
# you can do create_job+job_status with a single method
app
=
'
sleep
'
# the appli you want to execute
params
=
'
3
'
# sleep duration
files
=
None
# sleep do not use file, else create a list of filepath to load.
out_dict
=
client
.
run_job
(
app
,
params
=
params
,
files
=
files
,
verbose
=
True
)
print
(
out_dict
)
```
%% Output
running .
done
{'789': {'allgo.log': 'https://allgo18.inria.fr/datastore/789/allgo.log'}, 'status': 'done', 'id': 789}
%% Cell type:markdown id: tags:
### download a file generated
%% Cell type:code id: tags:
```
python
# last download a file generated
# here the log file.
wanted_filename
=
'
allgo.log
'
outdir
=
"
output_dir
"
print
(
"
We will download
'
{}
'
from job
'
{}
'
in
'
{}
'
.
"
.
format
(
wanted_filename
,
job_id
,
outdir
))
job_id
=
out_dict
[
'
id
'
]
url
=
out_dict
[
str
(
job_id
)][
'
allgo.log
'
]
filepath
=
client
.
download_file
(
url
,
force
=
True
,
outdir
=
outdir
)
print
(
"
downloaded file : {}
"
.
format
(
filepath
))
with
open
(
filepath
,
'
r
'
)
as
fe
:
for
line
in
fe
:
print
(
"
...
\t
{}
"
.
format
(
line
),
end
=
''
)
```
%% Output
We will download 'allgo.log' from job '787' in 'output_dir'.
downloaded file : output_dir/allgo.log
...
... This is app 'sleep' called with parameters '3'
...
... The workdir contains:
...
... total 0
... -rw-r--r-- 1 500 500 72 Dec 6 08:49 allgo.log
... POUET v1
... + sleep 3
...
... ==== ALLGO JOB SUCCESS ====
%% Cell type:markdown id: tags:
## Deal with errors
### try ... except
%% Cell type:code id: tags:
```
python
# Protect your code to catch potential exception raised.
try
:
# your code allgo
pass
except
ag
.
StatusError
as
e
:
print
(
"
status: {}, msg: {}
"
.
format
(
e
.
status_code
,
e
.
msg
))
except
ag
.
AllgoError
as
e
:
print
(
"
Error in *client API*
"
)
print
(
e
)
```
...
...
This diff is collapsed.
Click to expand it.
tests/test_allgo.py
+
88
−
0
View file @
63272da1
...
...
@@ -364,4 +364,92 @@ def test_run_job__args(mock_sleep, mock_status, mock_create, client, capsys):
assert
x_output
==
capsys
.
readouterr
().
out
# ------------------------------------------------
def
__mock_file
(
mock_get
):
file_content
=
"
fake line1.
\n
line2 is fake too.
\n
"
# -- mock
mock_get
.
return_value
=
Mock
(
status_code
=
200
)
mock_get
.
return_value
.
iter_content
.
return_value
=
\
iter
([
file_content
.
encode
()])
return
file_content
@pytest.mark.success
@patch
(
'
allgo.requests.get
'
)
def
test_download_file__default
(
mock_get
,
client
):
"""
The method returns the filepath where the file has been written.
"""
# -- mock
file_content
=
__mock_file
(
mock_get
)
# -- tests
filename
=
tempfile
.
mktemp
(
dir
=
''
)
file_url
=
"
https://whatever.fr/
"
+
filename
filepath
=
client
.
download_file
(
file_url
)
assert
"
./
"
+
filename
==
filepath
content
=
""
with
open
(
filepath
,
"
r
"
)
as
file_out
:
content
=
file_out
.
read
()
assert
file_content
==
content
# cleaning
os
.
remove
(
filepath
)
@pytest.mark.success
@patch
(
'
allgo.requests.get
'
)
def
test_download_file__outdir
(
mock_get
,
client
,
tmp_path
):
"""
The method returns the filepath where the file has been written.
"""
# -- mock
file_content
=
__mock_file
(
mock_get
)
# -- tests
filename
=
"
wanted_file
"
file_url
=
"
https://whatever.fr/
"
+
filename
filepath
=
client
.
download_file
(
file_url
,
outdir
=
tmp_path
)
assert
os
.
path
.
join
(
str
(
tmp_path
),
filename
)
==
filepath
content
=
""
with
open
(
filepath
,
"
r
"
)
as
file_out
:
content
=
file_out
.
read
()
assert
file_content
==
content
@pytest.mark.success
@patch
(
'
allgo.requests.get
'
)
def
test_download_file__force
(
mock_get
,
client
):
"""
The method returns the filepath where the file has been written.
"""
# -- mock
file_content
=
__mock_file
(
mock_get
)
# -- tests
# create the file
tmp_filepath
=
tempfile
.
NamedTemporaryFile
(
dir
=
'
.
'
)
filename
=
os
.
path
.
basename
(
tmp_filepath
.
name
)
file_url
=
"
https://whatever.fr/
"
+
filename
# default is to raise an exception if the file exists.
# ~ with pytest.raises(FileExistsError): # python3
with
pytest
.
raises
(
OSError
)
as
err_info
:
filepath
=
client
.
download_file
(
file_url
)
# default is force=False
err
=
err_info
.
value
assert
errno
.
EEXIST
==
err
.
errno
try
:
client
.
download_file
(
file_url
,
force
=
True
)
# ~ except FileExistsError: #python3
except
OSError
as
e
:
tmp_filepath
.
close
()
if
e
.
errno
==
errno
.
EEXIST
:
pytest
.
fail
(
"
download_file call with force=True failed.
"
)
# cleaning
tmp_filepath
.
close
()
# ------------------------------------------------
This diff is collapsed.
Click to expand it.
Preview
0%
Loading
Try again
or
attach a new file
.
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Save comment
Cancel
Please
register
or
sign in
to comment