Verified Commit 96269849 authored by BERTOT Luke's avatar BERTOT Luke Committed by SIMONIN Matthieu

Support for trusted client with Grid’5000

Internal certificate. In this mode users can pass the g5k_user argument
to most calls to specify which user the API call should be made as. In
cases where g5k_user is not specified API calls will be made as the
anonymous user whose access is limited to the Grid’5000 reference API.
In this mode python-grid5000 does not store any login information, so
g5k_user must be provided explicitly provided on every call that requires
one.
parent 24c5d9ea
Pipeline #120023 passed with stage
in 2 minutes and 39 seconds
......@@ -21,6 +21,14 @@ python3.6:
- tox -e py36
- tox -e pep8
python2.7:
image: python:2.7
stage: test
tags: [qlf-ci.inria.fr]
script:
- pip install tox
- tox -e py27 pep8
#### Entering th release zone
pages:
stage: deploy
......
......@@ -688,6 +688,56 @@ In [2]: # gk is your entry point
#+END_SRC
*** Using Grid'5000 client certificates
~python-grid5000~ can also be used as a trusted client with Grid'5000
internal certificate. In this mode users can pass the ~g5k_user~ argument
to most calls to specify which user the API call should be made as. In
cases where ~g5k_user~ is not specified API calls will be made as the
~anonymous~ user whose access is limited to the Grid'5000 reference API.
In this mode ~python-grid5000~ does not store any login information, so
~g5k_user~ most be provided explicitly provided on every call that requires
one.
#+BEGIN_SRC python :exports code :tangle examples/certificate.py
import logging
from grid5000 import Grid5000
logging.basicConfig(level=logging.DEBUG)
gk = Grid5000(
uri="https://api-ext.grid5000.fr/stable/",
sslcert="/path/to/ssl/certfile.cert",
sslkey="/path/to/ssl/keyfile.key"
)
gk.sites.list()
job = site.jobs.create({"name": "pyg5k",
"command": "sleep 3600"},
g5k_user = "auser1")
# Since the 'anonymous' user can not inspect jobs the following call will raise exception
# python-grid5000.exceptions.Grid5000AuthenticationError: 401 Unauthorized
job.refresh()
# Both following call work since any user can request info on any jobs.
job.refresh(g5k_user='auser1')
job.refresh(g5k_user='auser2')
# Some operations can only be performed by the jobs creator.
# The following call will raise exception
# pyg5k.exceptions.Grid5000DeleteError: 403 Unauthorized
job.delete(g5k_user='auser2')
# This call works as expected
job.delete(g5k_user='auser1')
#+END_SRC
* Appendix :noexport:
** How to export this file
......@@ -700,4 +750,3 @@ In [2]: # gk is your entry point
Do ~C-c C-v t~ or ~M-x
org-babel-tangle~. The scripts are available under available under ~examples~.
......@@ -253,7 +253,7 @@ Before starting, the file ``$HOME/.python-grid5000.yaml`` will be loaded.
site = gk.sites["rennes"]
job = site.jobs.create({"name": "pyg5k",
"command": "sleep 3600"})
"command": "sleep 3600"})
while job.state != "running":
job.refresh()
......@@ -287,8 +287,8 @@ Before starting, the file ``$HOME/.python-grid5000.yaml`` will be loaded.
site = gk.sites["rennes"]
job = site.jobs.create({"name": "pyg5k",
"command": "sleep 3600",
"types": ["deploy"]})
"command": "sleep 3600",
"types": ["deploy"]})
while job.state != "running":
job.refresh()
......@@ -298,7 +298,7 @@ Before starting, the file ``$HOME/.python-grid5000.yaml`` will be loaded.
print("Assigned nodes : %s" % job.assigned_nodes)
deployment = site.deployments.create({"nodes": job.assigned_nodes,
"environment": "debian9-x64-min"})
"environment": "debian9-x64-min"})
# To get SSH access to your nodes you can pass your public key
#
# from pathlib import Path
......@@ -358,8 +358,8 @@ Before starting, the file ``$HOME/.python-grid5000.yaml`` will be loaded.
site = gk.sites["rennes"]
job = site.jobs.create({"name": "pyg5k",
"command": "sleep 3600",
"resources": "slash_22=1+nodes=1"})
"command": "sleep 3600",
"resources": "slash_22=1+nodes=1"})
while job.state != "running":
job.refresh()
......@@ -371,8 +371,8 @@ Before starting, the file ``$HOME/.python-grid5000.yaml`` will be loaded.
# create acces for all ips in the subnet
access = site.storage["msimonin"].access.create({"ipv4": ip_network,
"termination": {"job": job.uid,
"site": site.uid}})
"termination": {"job": job.uid,
"site": site.uid}})
4.7 Vlan API
~~~~~~~~~~~~
......@@ -434,9 +434,9 @@ Before starting, the file ``$HOME/.python-grid5000.yaml`` will be loaded.
site = gk.sites["rennes"]
job = site.jobs.create({"name": "pyg5k",
"command": "sleep 3600",
"resources": "{type='kavlan'}/vlan=1+nodes=1",
"types": ["deploy"]})
"command": "sleep 3600",
"resources": "{type='kavlan'}/vlan=1+nodes=1",
"types": ["deploy"]})
while job.state != "running":
job.refresh()
......@@ -444,8 +444,8 @@ Before starting, the file ``$HOME/.python-grid5000.yaml`` will be loaded.
time.sleep(5)
deployment = site.deployments.create({"nodes": job.assigned_nodes,
"environment": "debian9-x64-min",
"vlan": job.resources_by_type["vlans"][0]})
"environment": "debian9-x64-min",
"vlan": job.resources_by_type["vlans"][0]})
while deployment.status != "terminated":
deployment.refresh()
......@@ -485,9 +485,9 @@ Before starting, the file ``$HOME/.python-grid5000.yaml`` will be loaded.
site = gk.sites["rennes"]
job = site.jobs.create({"name": "pyg5k",
"command": "sleep 3600",
"resources": "{type='kavlan'}/vlan=1+{cluster='paranoia'}nodes=1",
"types": ["deploy"]
"command": "sleep 3600",
"resources": "{type='kavlan'}/vlan=1+{cluster='paranoia'}nodes=1",
"types": ["deploy"]
})
while job.state != "running":
......@@ -555,16 +555,16 @@ For this example you’ll need ``matplotlib``, ``seaborn`` and ``pandas``.
value = timeserie.values
measurement = timeserie.uid
df = pd.concat([df, pd.DataFrame({
"timestamp": timestamp,
"value": value,
"measurement": [measurement]*len(timestamp)
"timestamp": timestamp,
"value": value,
"measurement": [measurement]*len(timestamp)
})])
sns.relplot(data=df,
x="timestamp",
y="value",
hue="measurement",
kind="line")
x="timestamp",
y="value",
hue="measurement",
kind="line")
plt.show()
4.9 More snippets
......@@ -594,8 +594,8 @@ For this example you’ll need ``matplotlib``, ``seaborn`` and ``pandas``.
candidates = site.clusters.list()
matching = [c.uid for c in candidates if c.uid in clusters]
if len(matching) == 1:
matches.append((site, matching[0]))
clusters.remove(matching[0])
matches.append((site, matching[0]))
clusters.remove(matching[0])
print("We found the following matches %s" % matches)
4.9.2 Get all job with a given name on all the sites
......@@ -625,13 +625,13 @@ For this example you’ll need ``matplotlib``, ``seaborn`` and ``pandas``.
jobs = []
for site in sites:
job = site.jobs.create({"name": "pyg5k",
"command": "sleep 3600"})
"command": "sleep 3600"})
jobs.append(job)
_jobs = []
for site in sites:
_jobs.append((site.uid, site.jobs.list(name=NAME,
state="waiting,launching,running")))
state="waiting,launching,running")))
print("We found %s" % _jobs)
......@@ -674,13 +674,13 @@ cross-processes cache) and give you control on the cached object. Enough talking
def get_api_client():
"""Gets the reference to the API cient (singleton)."""
with _api_lock:
global _api_client
if not _api_client:
conf_file = os.path.join(os.environ.get("HOME"),
".python-grid5000.yaml")
_api_client = Grid5000.from_yaml(conf_file)
global _api_client
if not _api_client:
conf_file = os.path.join(os.environ.get("HOME"),
".python-grid5000.yaml")
_api_client = Grid5000.from_yaml(conf_file)
return _api_client
return _api_client
@ring.disk(storage)
......@@ -696,8 +696,8 @@ cross-processes cache) and give you control on the cached object. Enough talking
sites = get_sites_obj()
clusters = []
for site in sites:
# should we cache the list aswell ?
clusters.extend(site.clusters.list())
# should we cache the list aswell ?
clusters.extend(site.clusters.list())
return clusters
......@@ -710,3 +710,52 @@ cross-processes cache) and give you control on the cached object. Enough talking
print("Calling again the function is now faster")
clusters = get_all_clusters_obj()
print(clusters)
4.9.4 Using Grid’5000 client certificates
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
``python-grid5000`` can also be used as a trusted client with Grid’5000
internal certificate. In this mode users can pass the ``g5k_user`` argument
to most calls to specify which user the API call should be made as. In
cases where ``g5k_user`` is not specified API calls will be made as the
``anonymous`` user whose access is limited to the Grid’5000 reference API.
In this mode ``python-grid5000`` does not store any login information, so
``g5k_user`` most be provided explicitly provided on every call that requires
one.
.. code:: python
import logging
from grid5000 import Grid5000
logging.basicConfig(level=logging.DEBUG)
gk = Grid5000(
uri="https://api-ext.grid5000.fr/stable/",
sslcert="/path/to/ssl/certfile.cert",
sslkey="/path/to/ssl/keyfile.key"
)
gk.sites.list()
job = site.jobs.create({"name": "pyg5k",
"command": "sleep 3600"},
g5k_user = "auser1")
# Since the 'anonymous' user can not inspect jobs the following call will raise exception
# python-grid5000.exceptions.Grid5000AuthenticationError: 401 Unauthorized
job.refresh()
# Both following call work since any user can request info on any jobs.
job.refresh(g5k_user='auser1')
job.refresh(g5k_user='auser2')
# Some operations can only be performed by the jobs creator.
# The following call will raise exception
# pyg5k.exceptions.Grid5000DeleteError: 403 Unauthorized
job.delete(g5k_user='auser2')
# This call works as expected
job.delete(g5k_user='auser1')
import logging
from grid5000 import Grid5000
logging.basicConfig(level=logging.DEBUG)
gk = Grid5000(
uri="https://api-ext.grid5000.fr/stable/",
sslcert="/path/to/ssl/certfile.cert",
sslkey="/path/to/ssl/keyfile.key"
)
gk.sites.list()
job = site.jobs.create({"name": "pyg5k",
"command": "sleep 3600"},
g5k_user = "auser1")
# Since the 'anonymous' user can not inspect jobs the following call will raise exception
# python-grid5000.exceptions.Grid5000AuthenticationError: 401 Unauthorized
job.refresh()
# Both following call work since any user can request info on any jobs.
job.refresh(g5k_user='auser1')
job.refresh(g5k_user='auser2')
# Some operations can only be performed by the jobs creator.
# The following call will raise exception
# pyg5k.exceptions.Grid5000DeleteError: 403 Unauthorized
job.delete(g5k_user='auser2')
# This call works as expected
job.delete(g5k_user='auser1')
......@@ -49,17 +49,20 @@ class Grid5000(object):
verify_ssl (bool); Whether SSL certificates should be validated.
timeout (float): Timeout to use for requests to the Grid5000 API.
session (requests.Session): session to use
ssl_cert (str): path to the client certificate file for Grid5000 API
ssl_key (str): path to the client key file for Grid5000 API
"""
def __init__(
self,
*,
uri=DEFAULT_BASE_URL,
username=None,
password=None,
verify_ssl=True,
timeout=None,
session=None,
sslcert=None,
sslkey=None,
**kwargs
):
self._uri = uri
......@@ -68,12 +71,21 @@ class Grid5000(object):
self.password = password
self.verify_ssl = verify_ssl
self.client_ssl = False
self.client_cert = None
if sslcert is not None:
self.client_ssl = True
if sslkey is not None:
self.client_cert = (sslcert, sslkey)
else:
self.client_cert = sslcert
self.headers = {"user-agent": USER_AGENT}
self.session = _create_session()
# manage auth
self._http_auth = None
if self.username:
if self.username and not self.client_ssl:
self._http_auth = requests.auth.HTTPBasicAuth(self.username, self.password)
self.root = RootManager(self)
......@@ -97,16 +109,14 @@ class Grid5000(object):
return cls(**conf)
except Exception as e:
logging.warn(e)
logging.info(
"...Falling back to anonymous connection"
)
logging.info("...Falling back to anonymous connection")
return cls()
def __enter__(self):
return self
def __exit__(self, *args):
return self.session.cloase()
return self.session.close()
def _build_url(self, path):
"""Returns the full url from path.
......@@ -122,26 +132,52 @@ class Grid5000(object):
else:
return "%s%s" % (self._uri, path)
def _get_session_opts(self, content_type=None, accept=None):
def _get_session_opts(
self, content_type=None, accept=None, user_id=None, other_headers=None
):
"""Returns list of option and headers to use of an http transaction
Args:
content_type (str): value of the Content-type http headers
accept (str) : value of the Accept http header
user_id (str) : Grid5000 user id to use in certificate mode
other_headers (dict) : other http headers to include
"""
request_headers = self.headers.copy()
if content_type is not None:
request_headers["Content-type"] = content_type
if accept is not None:
request_headers["Accept"] = accept
return {
res = {
"headers": request_headers,
"auth": self._http_auth,
"timeout": self.timeout,
"verify": self.verify_ssl,
"cert": self.client_cert,
}
if self.client_ssl:
if user_id is not None:
request_headers["X-Api-User-CN"] = user_id
request_headers["X-Remote-Ident"] = user_id
else:
request_headers["X-Api-User-CN"] = "anonymous"
request_headers["X-Remote-Ident"] = "anonymous"
else:
res["auth"] = self._http_auth
if other_headers is not None:
request_headers.update(other_headers)
return res
def http_request(
self,
verb,
path,
query_data=None,
post_data=None,
header_data=None,
streamed=False,
content_type="application/json",
accept="application/json",
......@@ -157,6 +193,7 @@ class Grid5000(object):
query_data (dict): Data to send as query parameters
post_data (dict): Data to send in the body (will be converted to
json)
header_data (dict): Data to send as http headers
streamed (bool): Whether the data should be streamed
**kwargs: Extra options to send to the server (e.g. sudo)
......@@ -169,10 +206,20 @@ class Grid5000(object):
query_data = {} if query_data is None else query_data
url = self._build_url(path)
opts = self._get_session_opts(content_type=content_type, accept=accept)
g5k_user = None
if self.client_ssl:
g5k_user = kwargs.pop("g5k_user", None)
opts = self._get_session_opts(
content_type=content_type,
accept=accept,
user_id=g5k_user,
other_headers=header_data,
)
verify = opts.pop("verify")
timeout = opts.pop("timeout")
cert = opts.pop("cert")
json = post_data
data = None
......@@ -186,12 +233,11 @@ class Grid5000(object):
prepped = req.prepare()
settings = self.session.merge_environment_settings(
prepped.url, {}, streamed, verify, None
prepped.url, {}, streamed, verify, cert
)
while True:
result = self.session.send(prepped, timeout=timeout, **settings)
# TODO:
# https://www.grid5000.fr/mediawiki/index.php/API#Status_Codes
if 200 <= result.status_code < 300:
......
......@@ -198,7 +198,7 @@ class ObjectDeleteMixin(object):
Grid5000AuthenticationError: If authentication is not correct
Grid5000DeleteError: If the server cannot perform the request
"""
self.manager.delete(self.get_id())
self.manager.delete(self.get_id(), **kwargs)
# Composite Mixins
......
......@@ -24,7 +24,7 @@ setup_requires =
install_requires =
requests>=2.21
pyyaml>=5.1
ipython>=7.3.0
ipython
[options.packages.find]
exclude =
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment