Add a caching section in the doc

parent 322dde07
Pipeline #81430 passed with stage
in 2 minutes
# ring cache
cachedir/
.pytest_cache
################
# Ansible #
......
......@@ -548,6 +548,78 @@ In [2]: # gk is your entry point
job.delete()
#+END_SRC
*** Caching API responses
The Grid'5000 reference API is static. In this situation to speed up the
requests, one could leverage heavily on caching. Currently
~python-grid5000~ doesn't do caching out-of the box but defers that to the
consuming application. There are many solutions to implement a cache.
Amongst them LRU cache
(https://docs.python.org/3/library/functools.html#functools.lru_cache)
provides an in-memory caching facilities but doesn't give you control on the
cache. The ring library (https://ring-cache.readthedocs.io/en/stable/) is
great as it implements different backends for your cache (esp.
cross-processes cache) and give you control on the cached object. Enough talking:
#+BEGIN_SRC python :exports code :tangle examples/caching.py
import logging
import threading
import os
import diskcache
from grid5000 import Grid5000
import ring
_api_lock = threading.Lock()
# Keep track of the api client
_api_client = None
storage = diskcache.Cache('cachedir')
def get_api_client():
"""Gets the reference to the API cient (singleton)."""
with _api_lock:
global _api_client
if not _api_client:
conf_file = os.path.join(os.environ.get("HOME"),
".python-grid5000.yaml")
_api_client = Grid5000.from_yaml(conf_file)
return _api_client
@ring.disk(storage)
def get_sites_obj():
"""Get all the sites."""
gk = get_api_client()
return gk.sites.list()
@ring.disk(storage)
def get_all_clusters_obj():
"""Get all the clusters."""
sites = get_sites_obj()
clusters = []
for site in sites:
# should we cache the list aswell ?
clusters.extend(site.clusters.list())
return clusters
if __name__ == "__main__":
logging.basicConfig(level=logging.DEBUG)
clusters = get_all_clusters_obj()
print(clusters)
print("Known key in the cache")
print(get_all_clusters_obj.get())
print("Calling again the function is now faster")
clusters = get_all_clusters_obj()
print(clusters)
#+END_SRC
* Appendix :noexport:
** How to export this file
......
......@@ -567,3 +567,75 @@ Before starting, the file ``$HOME/.python-grid5000.yaml`` will be loaded.
# deleting the jobs
for job in jobs:
job.delete()
4.8.3 Caching API responses
^^^^^^^^^^^^^^^^^^^^^^^^^^^
The Grid’5000 reference API is static. In this situation to speed up the
requests, one could leverage heavily on caching. Currently
``python-grid5000`` doesn’t do caching out-of the box but defers that to the
consuming application. There are many solutions to implement a cache.
Amongst them LRU cache
(`https://docs.python.org/3/library/functools.html#functools.lru_cache <https://docs.python.org/3/library/functools.html#functools.lru_cache>`_)
provides an in-memory caching facilities but doesn’t give you control on the
cache. The ring library (`https://ring-cache.readthedocs.io/en/stable/ <https://ring-cache.readthedocs.io/en/stable/>`_) is
great as it implements different backends for your cache (esp.
cross-processes cache) and give you control on the cached object. Enough talking:
.. code:: python
import logging
import threading
import os
import diskcache
from grid5000 import Grid5000
import ring
_api_lock = threading.Lock()
# Keep track of the api client
_api_client = None
storage = diskcache.Cache('cachedir')
def get_api_client():
"""Gets the reference to the API cient (singleton)."""
with _api_lock:
global _api_client
if not _api_client:
conf_file = os.path.join(os.environ.get("HOME"),
".python-grid5000.yaml")
_api_client = Grid5000.from_yaml(conf_file)
return _api_client
@ring.disk(storage)
def get_sites_obj():
"""Get all the sites."""
gk = get_api_client()
return gk.sites.list()
@ring.disk(storage)
def get_all_clusters_obj():
"""Get all the clusters."""
sites = get_sites_obj()
clusters = []
for site in sites:
# should we cache the list aswell ?
clusters.extend(site.clusters.list())
return clusters
if __name__ == "__main__":
logging.basicConfig(level=logging.DEBUG)
clusters = get_all_clusters_obj()
print(clusters)
print("Known key in the cache")
print(get_all_clusters_obj.get())
print("Calling again the function is now faster")
clusters = get_all_clusters_obj()
print(clusters)
import logging
import threading
import os
import diskcache
from grid5000 import Grid5000
import ring
_api_lock = threading.Lock()
# Keep track of the api client
_api_client = None
storage = diskcache.Cache('cachedir')
def get_api_client():
"""Gets the reference to the API cient (singleton)."""
with _api_lock:
global _api_client
if not _api_client:
conf_file = os.path.join(os.environ.get("HOME"),
".python-grid5000.yaml")
_api_client = Grid5000.from_yaml(conf_file)
return _api_client
@ring.disk(storage)
def get_sites_obj():
"""Get all the sites."""
gk = get_api_client()
return gk.sites.list()
@ring.disk(storage)
def get_all_clusters_obj():
"""Get all the clusters."""
sites = get_sites_obj()
clusters = []
for site in sites:
# should we cache the list aswell ?
clusters.extend(site.clusters.list())
return clusters
if __name__ == "__main__":
logging.basicConfig(level=logging.DEBUG)
clusters = get_all_clusters_obj()
print(clusters)
print("Known key in the cache")
print(get_all_clusters_obj.get())
print("Calling again the function is now faster")
clusters = get_all_clusters_obj()
print(clusters)
......@@ -25,15 +25,17 @@ while job.state != "running":
print("Assigned nodes : %s" % job.assigned_nodes)
deployment = site.deployments.create({"nodes": job.assigned_nodes,
# you can pass the content of your public
# key to get SSH access your nodes
#
# key: key_path.read_text()
#
# with
# from pathlib import Path
# key_path = Path.home().joinpath(".ssh", "id_rsa.pub")
"environment": "debian9-x64-min"})
# To get SSH access to your nodes you can pass your public key
#
# from pathlib import Path
#
# key_path = Path.home().joinpath(".ssh", "id_rsa.pub")
#
#
# deployment = site.deployments.create({"nodes": job.assigned_nodes,
# "environment": "debian9-x64-min"
# "key": key_path.read_text()})
while deployment.status != "terminated":
deployment.refresh()
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment