Jupyter token 2021

Update the allgo API in order to be able to manage the token of notebooks In notebooks, it should be call as :

• allgo.Init() # allgo_instance='https://allgo-qualif.inria.fr')
• app = allgo.App('Nd-Safir')
• app.run(...)

allgo/__init__.py

+#!/usr/bin/python3
+
+from distutils.version import StrictVersion
 import logging
 import os
 import time
+import secrets
+import pprint
+
+from IPython.display import HTML, display
 
 try:
     from urllib.parse import urlparse, urlencode
@@ -13,10 +20,25 @@ except ImportError:
 import requests
 
 log = logging.getLogger('allgo')
-__version__ = '0.1.11'
 
+# API version
+__version__ = StrictVersion('0.2.6')
+
+##################################################
+
+API_parameter_list = {'allgo_instance':'url','verbose_level':['errors', 'warnings', 'all'],'auth_method':['notebook','local_file','env_variable', 'arg_variable'],'timeout':'int value in sec'}
 
-def local_token():
+# timeout: automatic login default retry time in seconds
+# verbose_level: default verbose level
+# auth_method: notebook, local_file, env_variable, arg_variable
+# allgo_instance: The default Allgo url used
+API_parameters = {'allgo_instance':'https://allgo18.inria.fr','verbose_level':'warnings','auth_method':'notebook','timeout':60}
+
+# allgo API token validate on allgo plateform
+API_token_valid = False
+notebook_initialized = True
+
+def local_file_token():
     from os.path import expanduser
     home = expanduser("~")
     filetoken = os.path.join(home, '.allgo_token')
@@ -24,6 +46,195 @@ def local_token():
         with open(filetoken) as f:
             return f.read()
 
+def html_redbox(msg):
+    return HTML('<div style="width:80%; color:red;align:left;text-align:center;border-style: solid;padding:10px;">{}</div>'.format(msg))
+def html_redtxt(msg):
+    return HTML('<div style="color:red;text-align:left;">{}</div>'.format(msg))
+def html_greentxt(msg):
+    return HTML('<div style="color:green;text-align:left;">{}</div>'.format(msg))
+def html_url(url, txt):
+    return HTML('<a href={}>{}</div>'.format(url,txt))
+
+def print_file(file):
+    HTML(filename=file)
+
+class Init:
+    """
+    manage initialization of the API.
+    check if there is an existing token
+       | No: generate a temporary token and return the allgo login page
+       | Yes: Display "Allgo is already initialized"
+
+    accessors
+    | allgo_url - the url of the instance the user want to ask to.
+    | token_id     - the user token linked to the instance. cf {allgo_url}/profile
+    | verify_tls- does tls have to be verified ?
+
+        """
+
+
+    def __new__(self,  *args, **kwargs):
+        """
+        Constructor
+        :param name: name of the application in lower case
+        :param token: if not provided, we check ALLGO_TOKEN env variable and notebook parameters
+        """
+
+
+        global API_token_valid, API_parameter_list, API_parameters, notebook_initialized
+        notebook_initialized = True;
+        
+        if 'verbose' in kwargs:
+            if (self.check_parameters('verbose', kwargs['verbose'])):
+                API_parameters['verbose_level'] = kwargs['verbose']
+            else:
+                if(not API_parameters['verbose_level'] == 'errors'):
+                    print("\n 'verbose' restored to default: "+API_parameters['verbose_level'])
+                
+        if 'auth_method' in kwargs:
+            if (self.check_parameters('auth_method', kwargs['auth_method'])):
+                API_parameters['auth_method'] = kwargs['auth_method']
+            else:
+                if(not API_parameters['verbose_level'] == 'errors'):
+                    print("\n 'auth_method' restored to default: "+API_parameters['auth_method'])
+
+        if 'timeout' in kwargs:
+            if (isinstance(kwargs['timeout'], int)):
+                API_parameters['timeout'] = kwargs['timeout']
+            else:
+                if(not API_parameters['verbose_level'] == 'errors'):
+                    print("\n 'timeout' not an 'int' restored to default: "+str(API_parameters['timeout']))
+
+        if 'allgo_instance' in kwargs:
+            API_parameters['allgo_instance'] = kwargs['allgo_instance']
+
+        # Try to communicate with Allgo to test allgo instance
+        try:
+            if(not API_parameters['verbose_level'] == 'errors'):
+                print("\n Try to connect to the allgo API instance: "+API_parameters['allgo_instance'])
+            r = requests.get(API_parameters['allgo_instance']+str("/api/v1/notebook/validate_token/__faketoken__"), params=None,verify=True)
+            try:
+                r.raise_for_status()
+                results = r.json()
+                if 'status' in results.keys():
+                    status = results['status']
+                else:
+                    status = list(results.values())[0]['status']
+
+            except requests.exceptions.HTTPError as err:
+                if(API_parameters['verbose_level'] == 'errors'):
+                  print("Fail to connect to the allgo API instance: "+API_parameters['allgo_instance'])
+                if(not API_parameters['verbose_level'] == 'all'):
+                    print("Error: {}".format(str(err)))
+                notebook_initialized = False
+                return
+
+        except requests.exceptions.RequestException as err:  # This is the correct syntax
+            if(not API_parameters['verbose_level'] == 'none'):
+                print("Impossible to get the selected allgo API instance: test"+API_parameters['allgo_instance'])
+                print("Error: {}".format(str(err)))
+            notebook_initialized = False
+            return
+
+
+        # Check for other keywords not valid
+        new_dict = {k: kwargs[k] for k in kwargs if not k in API_parameter_list.keys()}
+        if len(new_dict) > 0:
+            print('')
+            print("Allgo.Init(): parameters are not taken in account.")
+            pprint.pprint(new_dict)
+            print("")
+            print("Valid parameters are: ")
+            pprint.pprint(API_parameter_list)
+            print("")
+
+        # Check all methods
+        # notebook, local_file, env_variable, arg_variable
+        if(API_parameters['auth_method'] == 'notebook'):
+            if 'API_token' not in API_parameters :
+                API_parameters['API_token'] = secrets.token_hex(16)
+                print('Notebook token created')
+            elif(API_token_valid):
+                print('Notebook token already created and valid for Allgo')
+            else:
+                print('Notebook token already created') # but not valid
+
+        elif(API_parameters['auth_method'] == 'local_file'):
+            API_parameters['API_token'] = self.local_file_token()
+            API_token_valid = True # Assume it is
+            
+        elif(API_parameters['auth_method'] == 'env_variable'):
+            if 'ALLGO_TOKEN' in os.environ.keys():
+                API_parameters['API_token'] = os.environ.get('ALLGO_TOKEN')
+                API_token_valid = True # Assume it is
+            
+        elif(API_parameters['auth_method'] == 'arg_variable'):
+            API_parameters['API_token']= token
+            API_token_valid = True # Assume it is
+        else:
+            print('Allgo init authentification method "'+str(API_parameters['auth_method'])+'" not recognized')
+
+    def check_parameters(param, kparam):
+        global API_parameter_list, API_parameters
+        if not kparam in API_parameter_list[param]:
+            if(not API_parameters['verbose_level'] == 'none'):
+                print("Allgo.Init(): Wrong value for '"+param+"'")
+                print("Valid parameters are: ")
+                pprint.pprint(API_parameter_list)
+                print(param+" value set to default: "+API_parameters[param])
+            return False
+        return True
+
+    def _request_api(self, method, url_end, **kwargs):
+        """Build and launch a http request on the Allgo instance API.
+
+        Parameters
+        ----------
+        method : string
+            'post' or 'get', the method to use in the http request.
+        url_end : string
+            end of the url to place after the API "signature".
+        kwargs : dict
+            any named parameters to use in the API request.
+
+        Returns
+        -------
+        dictionnary or Response
+            the API json response as a dictionnary if possible.
+            if not (like for downloading files), the requests Response object.
+
+        Raises
+        ------
+        StatusError
+            with the Response status code if it's not 200.
+        AllgoError
+            if the method is unknown.
+        """
+        headers = {'Authorization': 'Token token={}'.format(self.token)}
+        url = '{}/api/v1/{}'.format(self.allgo_url, url_end)
+
+        actions = {
+            'post': requests.post,
+            'get' : requests.get,
+        }
+        try:
+            resp = actions[method](url, headers=headers, verify=self.verify_tls, **kwargs)
+            logging.debug("raw API answer : %s", str(resp.content))
+            if resp.status_code != requests.codes.ok:
+                logging.debug("req.status_code = %s", str(resp.status_code))
+                raise StatusError(resp.status_code, resp)
+            return resp.json()
+        except ValueError:  # pure python3 should use JSONDecodeError
+            # this happens when response content cannot be converted to json
+            #   for example when downloading a file.
+            return resp
+        except KeyError as err:
+            msg  = "method {} is not allowed to request the API.".format(method)
+            msg += "KeyError: {}".format(str(err))
+            
+            return display(html_redbox(msg))
+
+
 
 class App:
     """
@@ -34,19 +245,66 @@ class App:
         """
         Constructor
         :param name: name of the application in lower case
-        :param token: if not provided, we check ALLGO_TOKEN env variable and notebook parameters
+        :param token: if not provided, we check internal notebook token value. If not, check ALLGO_TOKEN env variable
         """
+        global API_token_valid, API_parameters, notebook_initialized
+
+        if (not notebook_initialized):
+            print("Initialisation failed, please check your parameters. You must call Init() first")
+            return
+
         self.name = name
-        if token:
-            self.token = token
-        elif 'ALLGO_TOKEN' in os.environ.keys():
-            self.token = os.environ.get('ALLGO_TOKEN')
-        elif local_token():
-            self.token = local_token()
+#        if API_token_valid:
+            # notebook token valid or other mean already initialized
+        if not API_token_valid and API_parameters['auth_method']=='notebook':
+            retries = 0
+            sleep_time = 2 # in seconds
+            retry = API_parameters['timeout']/2 # in seconds
+            display(html_redtxt("You need first to <a href=\'"+API_parameters['allgo_instance']+"/notebook?token=" +API_parameters['API_token']+'\' target=\"_blank\">sign in with your existing A||GO account</a> ('+str(API_parameters['timeout'])+' seconds timeout)'))
+            print("Waiting for allgo login .", end="")
+            while retries < retry :
+                retries += 1
+                r = requests.get(API_parameters['allgo_instance']+str("/api/v1/notebook/validate_token/")+str(API_parameters['API_token']), params=None,verify=True)
+                r.raise_for_status()
+                results = r.json()
+                if 'status' in results.keys():
+                    status = results['status']
+                else:
+                    status = list(results.values())[0]['status']
+
+                if status in ['valid']:
+                    API_token_valid = True
+                    retry = 0
+                else:
+                    print('.', end="")
+                time.sleep(sleep_time)
+
+        # Check login ?
+        if API_token_valid:
+            display(html_greentxt('Notebook now linked to Allgo'))
+
+            # Check application name
+            # Post a fake job
+            headers = {'Authorization': 'Token token={}'.format(API_parameters['API_token'])}
+            data = {"job[webapp_name]": self.name,
+                    "job[webapp_id]": self.name,
+                    "job[param]": None}
+            url = API_parameters['allgo_instance'] + '/api/v1/jobs'
+
+            response = requests.post(url, headers=headers, files=None, data=data, verify=True)
+            # if the name is not good
+            if not response:
+                display(html_redtxt("Application "+name+" does not exist "))
+                return
+
+            response_json = response.json()
+            if 'error' in response_json.keys():
+                display(html_redtxt("Application "+name+": "+response_json['error']))
+            else:
+                display(html_greentxt('You could use '+name+' or any other Allgo application'))
+
         else:
-            err_msg  = "You must provide a token in parameter"
-            err_msg += " or define an environment variable 'ALLGO_TOKEN'"
-            raise Exception(err_msg)
+            display(html_redtxt('Allgo notebook token failed to initialized, please replay your notebook'))
 
     def run(self, files, outputdir='.', params='', verify_tls=True):
         """
@@ -57,22 +315,32 @@ class App:
         :param verify_tls: [True] the value is pass to the verify arg of requests.post
         :return:
         """
-        headers = {'Authorization': 'Token token={}'.format(self.token)}
+        global API_parameters
+        headers = {'Authorization': 'Token token={}'.format(API_parameters['API_token'])}
         data = {"job[webapp_name]": self.name,
                 "job[webapp_id]": self.name,
                 "job[param]": params}
-        ALLGO_URL = os.environ.get('ALLGO_URL', "https://allgo.inria.fr")
-        url = '%s/api/v1/jobs' % ALLGO_URL
-        r = requests.post(url, headers=headers, files=files, data=data, verify=verify_tls)
-        r.raise_for_status()
-        r = r.json()
-        if 'id' in r.keys():
-            jobid = r['id']
+        url = API_parameters['allgo_instance'] + '/api/v1/jobs'
+
+        response = requests.post(url, headers=headers, files=files, data=data, verify=verify_tls)
+
+        response = response.json()
+        if 'id' in response.keys():
+            jobid = response['id']
         else:
-            jobid = list(r.keys())[0]
+            jobid = list(response.keys())[0]
         results = None
+        # API result format
+        #{
+        # "40155":
+        #   {"conv1_samusa.txt":"https://allgo18.inria.fr/datastore/6/1/0fe2bc68b835e9a1f681e38d5e87001ef955e345/conv1_samusa.txt",
+        #    "allgo.log":"https://allgo18.inria.fr/datastore/6/1/0fe2bc68b835e9a1f681e38d5e87001ef955e345/allgo.log"
+        #    },
+        # "status":"done"
+        # }
+        previous_status = ''
         while True:
-            url = '{}/api/v1/jobs/{}'.format(ALLGO_URL, jobid)
+            url = '{}/api/v1/jobs/{}'.format(API_parameters['allgo_instance'], jobid)
             r = requests.get(url, headers=headers, verify=verify_tls)
             r.raise_for_status()
             results = r.json()
@@ -80,25 +348,30 @@ class App:
                 status = results['status']
             else:
                 status = list(results.values())[0]['status']
+
             if status in ['created', 'waiting', 'running', 'in progress']:
-                log.info("wait for job %s in status %s", jobid, status)
+                if (previous_status != status):
+                    print('')
+                    print('Job '+str(jobid)+' status: '+status+" ", end="")
+                    previous_status = status
+                else:
+                    print('.', end="")
                 time.sleep(2)
             else:
                 break
-
         if status != 'done':
-            raise Exception('Job %s failed with status %s', (jobid, status))
+            return display(html_redtxt('ERROR : Job '+str(jobid)+' failed with status '+status))
 
         elif status == 'done' and results:
-            if 'id' in results.keys():
-                files = results[str(jobid)].items()
-            else:
-                files = results[str(jobid)]['files'].items()
+            files = results[str(jobid)].items()
+            print('')
+            print('Get file for job '+str(jobid)+':')
             for filename, url in files:
                 filepath = os.path.join(outputdir, filename)
-                with requests.get(url, headers=headers, verify=verify_tls, stream=True) as r:
-                    r.raise_for_status()
-                    with open(filepath, 'wb') as f:
-                        for chunk in r.iter_content(chunk_size=8192): 
-                            if chunk: 
-                                f.write(chunk)
+                print(filepath+' ')
+                r = requests.get(url, headers=headers, verify=verify_tls, stream=True)
+                r.raise_for_status()
+                with open(filepath, 'wb') as f:
+                    for chunk in r.iter_content(chunk_size=8192):
+                        if chunk:
+                            f.write(chunk)