Mentions légales du service

Skip to content
Snippets Groups Projects
Commit 63272da1 authored by LETORT Sebastien's avatar LETORT Sebastien
Browse files

New method to download files, in python2 and python3.

parent e7a6810e
Branches
Tags
1 merge request!4Client api
......@@ -45,6 +45,7 @@ and finally :
# standard libs
from __future__ import print_function
import errno
import logging
import os
import sys
......@@ -346,6 +347,114 @@ class Client:
# --------------------------------------------
# -- Accessory methods, to ease user work.
# --------------------------------------------
def _request_file(self, file_url):
"""Build and launch a http request on any URL, but with Allgo authentication.
The file downloads as part of the API, it is an additionnal service.
Parameters
----------
file_url : string/URL
the URL to download the file from.
Returns
-------
filepath
the local filepath of the file downloaded.
Raises
------
StatusError
with the Response status code if it's not 200.
"""
headers = {'Authorization': 'Token token={}'.format(self.token)}
resp = requests.get(file_url, headers=headers, verify=self.verify_tls)
if resp.status_code != requests.codes.ok:
raise StatusError(resp.status_code, resp)
return resp
def download_file(self, *args, **kwargs):
"""download the file pointed by the url.
The file is downloaded in the outdir directory (which is created if not exist).
The old Allgo site (allgo.inria.fr) doesn't need authentication.
The new allgo site (allgo18.inria.fr) does need authentication.
Parameters
----------
file_url : string/URL
the URL to download the file from.
outdir : dirpath, optional
the dirpath to write file to.
force : bool, optional
set to true to force erasing existing file.
Returns
-------
filepath
the local filepath of the file downloaded.
Raises
------
FileExistsError
if the output file exists and force is false.
StatusError
with the Response status code if it's not 200.
"""
if 2 == sys.version_info.major:
return self.__download_file_p2(*args, **kwargs)
return self.__download_file_p3(*args, **kwargs)
def __download_file_p3(self, file_url, outdir='.', force=False):
mode = 'wb' if force else 'xb'
resp = self._request_file(file_url)
# urlparse is better, but a pain to use in python2 & 3
# fileurl, if come from job_status doesn't have params, so it's ok.
filename = os.path.basename(file_url)
os.makedirs(outdir, exist_ok=True)
outfilepath = os.path.join(outdir, filename)
with open(outfilepath, mode) as file_out:
# Note: '8192' = 2^13 : size of cache/buffer.
# = max size transfered in one operation.
for chunk in resp.iter_content(chunk_size=8192):
if chunk:
file_out.write(chunk)
return outfilepath
def __download_file_p2(self, file_url, outdir='.', force=False):
# python2: outdir cannot be a PosixPath (cf makedirs, os.path.join)
outdir = str(outdir)
filename = os.path.basename(file_url)
outfilepath = os.path.join(outdir, filename)
# method with os.fdopen(os.open(outfilepath, mode), 'w') doesn't exist
if os.path.isfile(outfilepath) and not force:
msg = "file {} exists."
raise OSError(errno.EEXIST, msg)
try:
os.makedirs(outdir)
except OSError as e:
# silence directory exists error
if e.errno != errno.EEXIST:
raise
resp = self._request_file(file_url)
with open(outfilepath, 'wb') as file_out:
# Note: '8192' = 2^13 : size of cache/buffer.
# = max size transfered in one operation.
for chunk in resp.iter_content(chunk_size=8192):
if chunk:
file_out.write(chunk)
return outfilepath
def run_job(self, app, version=None, params='', files=None,
sleep_duration=2, verbose=False):
"""Create a job and wait for it to terminate.
......
%% Cell type:markdown id: tags:
# Allgo Python API-client demonstrator
## Introduction
### reminder
* Allgo project : https://allgo18.inria.fr
* Allgo API documentation legacy : https://allgo.inria.fr/documentation/api
* Allgo API documentation : https://allgo.gitlabpages.inria.fr/doc/api.html
* API-client project : https://gitlab.inria.fr/allgo/api-clients
### this is a python demonstrator
Here we will show you how to use the python client to launch allgo-job.
## CODE :)
### import allgo
%% Cell type:code id: tags:
``` python
# first import allgo
import allgo
# you can check
print(allgo.__version__)
# print(allgo.__file__)
```
%% Output
0.2.0
%% Cell type:markdown id: tags:
### constructor - an allgo object represents the user-instance connexion
%% Cell type:code id: tags:
``` python
# you could specify an alternative url
token = None # put your token.
url = "https://localhost/" # here this is our dev instance.
client = allgo.Client(token, allgo_url=url, verify_tls=False) # on localhost we cannot verify tls.
print("You've got a connexion to {}".format(client.allgo_url))
```
%% Output
You've got a connexion to https://localhost/
%% Cell type:code id: tags:
``` python
# by default, it's the "official" instance that is used.
token = None # put your token.
client = allgo.Client(token)
print("You've got a connexion to {}".format(client.allgo_url))
```
%% Output
You've got a connexion to https://allgo18.inria.fr
%% Cell type:markdown id: tags:
### create a job
%% Cell type:code id: tags:
``` python
app = 'sleep' # the appli you want to execute
params = '3' # sleep duration
files = None # sleep do not use file, else create a list of filepath to load.
# your code allgo
out_dict = client.create_job(app, params=params, files=files)
job_id = out_dict['id']
print("=> Jobs launched with Id {}.".format(job_id))
print("full output dictionary : {}".format(out_dict))
```
%% Output
=> Jobs launched with Id 787.
full output dictionary : {'url': 'https://allgo18.inria.fr/api/v1/jobs/787', 'avg_time': 0, 'id': 787}
%% Cell type:markdown id: tags:
### get job status
%% Cell type:code id: tags:
``` python
# wait for the job to terminate
# = sleep until its status tell it's terminated.
from time import sleep
while True:
out_dict = client.job_status(job_id)
print( "out_dict = {}".format( out_dict ))
if out_dict['status'] != 'done':
print("... not done, let's wait 2 more sec.")
sleep(2)
else:
print("=> job's done !")
break
```
%% Output
out_dict = {'787': {'allgo.log': 'https://allgo18.inria.fr/datastore/787/allgo.log'}, 'status': 'done'}
=> job's done !
%% Cell type:markdown id: tags:
### trick to run a job in a single call
%% Cell type:code id: tags:
``` python
# you can do create_job+job_status with a single method
app = 'sleep' # the appli you want to execute
params = '3' # sleep duration
files = None # sleep do not use file, else create a list of filepath to load.
out_dict = client.run_job(app, params=params, files=files, verbose=True)
print(out_dict)
```
%% Output
running .
done
{'789': {'allgo.log': 'https://allgo18.inria.fr/datastore/789/allgo.log'}, 'status': 'done', 'id': 789}
%% Cell type:markdown id: tags:
### download a file generated
%% Cell type:code id: tags:
``` python
# last download a file generated
# here the log file.
wanted_filename = 'allgo.log'
outdir = "output_dir"
print("We will download '{}' from job '{}' in '{}'.".format(wanted_filename, job_id, outdir))
job_id = out_dict['id']
url = out_dict[str(job_id)]['allgo.log']
filepath = client.download_file(url, force=True, outdir=outdir)
print("downloaded file : {}".format(filepath))
with open(filepath, 'r') as fe:
for line in fe:
print("...\t{}".format(line), end='')
```
%% Output
We will download 'allgo.log' from job '787' in 'output_dir'.
downloaded file : output_dir/allgo.log
...
... This is app 'sleep' called with parameters '3'
...
... The workdir contains:
...
... total 0
... -rw-r--r-- 1 500 500 72 Dec 6 08:49 allgo.log
... POUET v1
... + sleep 3
...
... ==== ALLGO JOB SUCCESS ====
%% Cell type:markdown id: tags:
## Deal with errors
### try ... except
%% Cell type:code id: tags:
``` python
# Protect your code to catch potential exception raised.
try:
# your code allgo
pass
except ag.StatusError as e:
print("status: {}, msg: {}".format(e.status_code, e.msg))
except ag.AllgoError as e:
print("Error in *client API*")
print(e)
```
......
......@@ -364,4 +364,92 @@ def test_run_job__args(mock_sleep, mock_status, mock_create, client, capsys):
assert x_output == capsys.readouterr().out
# ------------------------------------------------
def __mock_file(mock_get):
file_content = "fake line1.\nline2 is fake too.\n"
# -- mock
mock_get.return_value = Mock(status_code=200)
mock_get.return_value.iter_content.return_value = \
iter([file_content.encode()])
return file_content
@pytest.mark.success
@patch('allgo.requests.get')
def test_download_file__default(mock_get, client):
"""The method returns the filepath where the file has been written."""
# -- mock
file_content = __mock_file(mock_get)
# -- tests
filename = tempfile.mktemp(dir='')
file_url = "https://whatever.fr/" + filename
filepath = client.download_file(file_url)
assert "./"+filename == filepath
content = ""
with open(filepath, "r") as file_out:
content = file_out.read()
assert file_content == content
# cleaning
os.remove(filepath)
@pytest.mark.success
@patch('allgo.requests.get')
def test_download_file__outdir(mock_get, client, tmp_path):
"""The method returns the filepath where the file has been written."""
# -- mock
file_content = __mock_file(mock_get)
# -- tests
filename = "wanted_file"
file_url = "https://whatever.fr/" + filename
filepath = client.download_file(file_url, outdir=tmp_path)
assert os.path.join(str(tmp_path), filename) == filepath
content = ""
with open(filepath, "r") as file_out:
content = file_out.read()
assert file_content == content
@pytest.mark.success
@patch('allgo.requests.get')
def test_download_file__force(mock_get, client):
"""The method returns the filepath where the file has been written."""
# -- mock
file_content = __mock_file(mock_get)
# -- tests
# create the file
tmp_filepath = tempfile.NamedTemporaryFile(dir='.')
filename = os.path.basename(tmp_filepath.name)
file_url = "https://whatever.fr/" + filename
# default is to raise an exception if the file exists.
# ~ with pytest.raises(FileExistsError): # python3
with pytest.raises(OSError) as err_info:
filepath = client.download_file(file_url) # default is force=False
err = err_info.value
assert errno.EEXIST == err.errno
try:
client.download_file(file_url, force=True)
# ~ except FileExistsError: #python3
except OSError as e:
tmp_filepath.close()
if e.errno == errno.EEXIST:
pytest.fail("download_file call with force=True failed.")
# cleaning
tmp_filepath.close()
# ------------------------------------------------
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Please register or to comment