diff --git a/docs/apidoc/api.rst b/docs/apidoc/api.rst index fe7834327251f3ad98008e17ec62286a8d891c12..d820e14671d9d7b1ab119431866865735c6af084 100644 --- a/docs/apidoc/api.rst +++ b/docs/apidoc/api.rst @@ -1,3 +1,7 @@ +API +=== + .. automodule:: distem.api :members: :undoc-members: + diff --git a/docs/index.rst b/docs/index.rst index 08886b570d38d7985e75d2675e0891d95adc79fd..91709e1c6ca31ab2b5d2dc34d60954fce89a4496 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -1,11 +1,8 @@ -.. enoslib documentation master file, created by - sphinx-quickstart on Thu Sep 21 21:45:39 2017. - You can adapt this file completely to your liking, but it should at least - contain the root `toctree` directive. +Welcome to Distem's documentation ! .. toctree:: - tutorials.rst + :maxdepth: 2 + :caption: Contents: -.. automodule:: distem - :members: - :undoc-members: + tutorials/index.rst + apidoc/index.rst diff --git a/docs/tutorial.rst b/docs/tutorial.rst deleted file mode 100644 index 4ced36388f8a23d9bb1727de28e99f12b9254f45..0000000000000000000000000000000000000000 --- a/docs/tutorial.rst +++ /dev/null @@ -1,29 +0,0 @@ -.. _tutorials: - -Provider::Distem -================ - -This tutorial show how the use of Distem, a provider that prepare containers for you on Grid'5000. - -Prerequisites -------------- - -First of all, you have to reserve some nodes on Grid'5000, deploy a system on the nodes and then use the distem bootstrap. - -.. code-block: bash - - $ oarsub -t deploy -l "slash_22=1+nodes=2, walltime=02:00:00" -I" - $ kadeploy3 -f $OAR_NODE_FILE -e debian9-x64-nfs -k - $ distem-bootstrap --debian-version stretch - -Get image for tutorial ----------------------- - -.. code-block: bash - - $ frontend> wget 'http://public.nancy.grid5000.fr/~amerlin/distem/distem-fs-jessie.tar.gz' -P ~/distem_img - -.. literalinclude:: scp_vsync/scp_vsync.py - :language: python - :linenos: - diff --git a/docs/tutorials/index.rst b/docs/tutorials/index.rst index 078008a45d3df917724e9fd0d4a755fe08c058b6..58c4ad8bcbcd010ae4d1a8e7a24175f50004042c 100644 --- a/docs/tutorials/index.rst +++ b/docs/tutorials/index.rst @@ -1,6 +1,44 @@ -.. toctree:: - :maxdepth: 2 - :caption Contents: +Provider::Distem +================ - tutorials - apidoc +This tutorial show how the use of Distem, a provider that prepare containers for you on Grid'5000. + +Prerequisites +------------- + +First of all, you have to reserve some nodes on Grid'5000, deploy a system on the nodes and then use the distem bootstrap. + +.. code-block: bash + + $ oarsub -t deploy -l "slash_22=1+nodes=2, walltime=02:00:00" -I" + $ kadeploy3 -f $OAR_NODE_FILE -e debian9-x64-nfs -k + $ distem-bootstrap --debian-version stretch + +Get image for tutorial +---------------------- + +You have to download the image in your home directory. + +.. code-block: bash + + $ frontend> wget 'http://public.nancy.grid5000.fr/~amerlin/distem/distem-fs-jessie.tar.gz' -P ~/distem_img + +Initialisation +-------------- + +To process to any experiments, you need to initialize nodes as the folowing code. + +.. literalinclude:: scp_vsync/initialisation.py + :language: python + :linenos: + +Experiment +---------- + +The experiment can be launched from the coordinator node as root. +Below, the python code of the experiment. + +.. literalinclude:: scp_vsync/experience.py + :language: python + :lineos: + diff --git a/examples/example.py b/examples/example.py deleted file mode 100644 index 622deb90d0e1b6a63e5a2c8cb59a1b94c816a10c..0000000000000000000000000000000000000000 --- a/examples/example.py +++ /dev/null @@ -1,185 +0,0 @@ -import time -import os -from distem import Distem - - -if __name__ == "__main__": - - # The path to the compressed filesystem image - # We can point to local file since our homedir is available from NFS - FSIMG = "file:///home/rolivo/distem_img/distem-fs-jessie.tar.gz" - # Put the physical machines that have been assigned to you - # You can get that by executing: cat $OAR_NODE_FILE | uniq - PNODES = ["econome-7.nantes.grid5000.fr", "econome-8.nantes.grid5000.fr"] - - # The first argument of the script is the address (in CIDR format) - # of the virtual network to set-up in our platform - # This ruby hash table describes our virtual network - vnet = { - 'name': 'mynetwork', - 'address': "10.176.0.0/22" - } - nodelist = ['node-1', 'node-2'] - # Read SSH keys - PRIV_KEY = os.path.join(os.environ["HOME"], ".ssh", "id_rsa") - PUB_KEY = "%s.pub" % PRIV_KEY - - private_key = open(os.path.expanduser(PRIV_KEY)).read() - public_key = open(os.path.expanduser(PUB_KEY)).read() - - sshkeys = { - "public" : public_key, - "private" : private_key - } - latencies = ['0ms', '20ms', '40ms', '60ms'] - results = { - 'scp': {}, - 'rsync': {} - } - avg = { - 'scp': {}, - 'rsync': {} - } - node1 = { - 'name': 'node-1', - 'address': [] - } - node2 = { - 'name': 'node-2', - 'address': [] - } - ifname = 'if0' - iteration = 5 - - def average(values): - sum_values = 0.0 - for i in values: - sum_values += float(i) - return sum_values / len(values) - - def stddev(values, avg): - sum_values = 0.0 - for i in values: - sum_values += (float(i) - avg) * (float(i) - avg) - return sum_values / len(values) - - # Connect to the Distem server (on http://localhost:4567 by default) - distem = Distem() - - # Start by creating the virtual network - print("Network created") - distem.vnetwork_create(vnet['name'], - vnet['address']) - # Creating one virtual node per physical one - # Create the first virtual node and set it to be hosted on - # the first physical machine - print("Node %s created" %nodelist[0]) - distem.vnode_create(nodelist[0], - {'host': PNODES[0]}, sshkeys) - # Specify the path to the compressed filesystem image - # of this virtual node - distem.vfilesystem_create(nodelist[0], - {'image': FSIMG}) - # Create a virtual network interface and connect it to vnet - print("Interface created") - distem.viface_create(nodelist[0], - 'if0', - {'vnetwork': vnet['name'], 'default': 'true'}) - # Create the first virtual node and set it to be hosted on - # the second physical machine - print("Node %s created" %nodelist[1]) - distem.vnode_create(nodelist[1], - {'host': PNODES[1]}, - sshkeys) - distem.vfilesystem_create(nodelist[1], - {'image': FSIMG}) - - # Create a virtual network interface and connect it to vnet - print("Interface created") - distem.viface_create(nodelist[1], - 'if0', - {'vnetwork': vnet['name']}) - # Starting the virtual nodes using the synchronous method - distem.vnodes_start(nodelist) - print("Nodes started") - - # Getting the -automatically affected- address of each virtual nodes - # virtual network interfaces - node1['address'] = distem.viface_info(node1['name'], ifname)['address'].split("/")[0] - node2['address'] = distem.viface_info(node2['name'], ifname)['address'].split("/")[0] - - # Creating the files we will use in our experimentation - distem.vnode_execute(node1['name'], - 'mkdir -p /tmp/src ; cd /tmp/src ; \ - for i in `seq 1 100`; do \ - dd if=/dev/zero of=$i bs=1K count=50; \ - done' - ) - - # Printing the current latency - start_time = float(time.time()) - distem.vnode_execute(node1['name'], 'hostname') - print("Latency without any limitations # %s" %(str(time.time() - start_time))) - - desc = { - 'output': { - 'latency': { - 'delay': [] - } - } - } - # Starting our experiment for each specified latencies - print('Starting tests') - for latency in latencies: - results['scp'][latency] = [] - results['rsync'][latency] = [] - avg['scp'][latency] = [] - avg['rsync'][latency] = [] - - print("Latency #%s" %str(latency)) - # Update the latency description on virtual nodes - desc['output']['latency']['delay'] = latency - distem.viface_update(node1['name'], ifname, desc) - distem.viface_update(node2['name'], ifname, desc) - - for i in range(iteration): - print("\tIteration ## %s" %i) - # Launch SCP test - # Cleaning target directory on node2 - distem.vnode_execute(node2['name'], 'rm -rf /tmp/dst') - # Starting the copy from node1 to node2 - start_time = time.time() - distem.vnode_execute(node1['name'], - "scp -rq /tmp/src #{node2['address']}:/tmp/dst" - ) - results['scp'][latency] = str(time.time() - start_time) - # Launch RSYNC test - # Cleaning target directory on node2 - distem.vnode_execute(node2['name'], 'rm -rf /tmp/dst') - # Starting the copy from node1 to node2 - start_time = round(time.time(), 5) - distem.vnode_execute('node-1', - "rsync -r /tmp/src #{node2['address']}:/tmp/dst" - ) - results['rsync'][latency] = str(time.time() - start_time) - - values = [] - print("Rsync results :") - for latency in latencies: - values.append(results['rsync'][latency]) - avg['rsync'][latency] = average(values) - - for latency in latencies: - print("%s: [average=%s,standard_deviation=%s]" - %(str(latency), avg['rsync'][latency], stddev(values, avg['rsync'][latency]))) - - values = [] - print("SCP results :") - for latency in latencies: - values.append(results['scp'][latency]) - avg['scp'][latency] = average(values) - - for latency in latencies: - print("%s: [average=%s,standard_deviation=%s]" - %(str(latency), avg['scp'][latency], stddev(values, avg['scp'][latency]))) -