Commit 0d1b3cd7 authored by Lucas Bourneuf's avatar Lucas Bourneuf

handle options for scripts + working examples

parent 8a872b0e
...@@ -3,8 +3,11 @@ ...@@ -3,8 +3,11 @@
.cache/ .cache/
.pytest_cache .pytest_cache
__pycache__/ __pycache__/
build/
biseau.egg-info/ biseau.egg-info/
build/
dist/ dist/
extracted.lp
extraction.lp
out.png
venv-*/ venv-*/
venv/ venv/
VERBOSITY=
# VERBOSITY=-v
# VERBOSITY=-vv
# VERBOSITY=-vvv
run-test: run-test:
python -m biseau scripts/example.lp scripts/black_theme.json -o out/out.png python -m biseau scripts/example.lp scripts/black_theme.json -o out/out.png $(VERBOSITY)
run-config-test:
python -m biseau -c ./configs/simple.json $(VERBOSITY)
python -m biseau -c ./configs/concept-lattice.json -o out/concept-lattice.png $(VERBOSITY)
xdg-open out/concept-lattice.png
t: test t: test
test: test:
......
"""Entry point for package """Entry point for package.
To play with scripts parameters, you must use the configuration
file instead of the direct enumeration of filenames.
""" """
import os import os
import argparse import argparse
import itertools
from . import core from . import core
...@@ -11,12 +15,15 @@ def parse_cli(args:iter=None) -> dict: ...@@ -11,12 +15,15 @@ def parse_cli(args:iter=None) -> dict:
# main parser # main parser
parser = argparse.ArgumentParser(description=__doc__) parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument('infiles', type=str, nargs='+', metavar='MODULE', parser.add_argument('infiles', type=str, nargs='*', metavar='MODULE',
default=[], help='files containing ASP or Python code') default=[], help='files containing ASP or Python code')
parser.add_argument('--outfile', '-o', type=str, default='out.png', parser.add_argument('--outfile', '-o', type=str, default='out.png',
help="output file. Will be overwritten with png data. Can be templated with '{model_number}'") help="output file. Will be overwritten with png data. Can be templated with '{model_number}'")
parser.add_argument('--dotfile', '-d', type=str, default=None, parser.add_argument('--dotfile', '-d', type=str, default=None,
help="output file. Will be overwritten with dot data. Can be templated with '{model_number}'") help="output file. Will be overwritten with dot data. Can be templated with '{model_number}'")
parser.add_argument('--config', '-c', type=str, default=None,
help="configuration file, specifying scripts and their options")
parser.add_argument('-v', '--verbosity', action='count', default=0)
# flags # flags
parser.add_argument('--flag-example', action='store_true', parser.add_argument('--flag-example', action='store_true',
...@@ -27,9 +34,16 @@ def parse_cli(args:iter=None) -> dict: ...@@ -27,9 +34,16 @@ def parse_cli(args:iter=None) -> dict:
if __name__ == '__main__': if __name__ == '__main__':
args = parse_cli() args = parse_cli()
all_infiles = args.infiles
if args.config:
all_infiles = itertools.chain(
all_infiles,
core.build_pipeline.from_configfile(args.config, verbosity=args.verbosity)
)
core.single_image_from_filenames( core.single_image_from_filenames(
args.infiles, all_infiles,
dotfile=args.dotfile, dotfile=args.dotfile,
outfile=args.outfile, outfile=args.outfile,
return_image=False, return_image=False,
verbosity=args.verbosity,
) )
...@@ -77,7 +77,7 @@ def visual_config_from_atoms(atoms:dict, base_atoms:dict, ...@@ -77,7 +77,7 @@ def visual_config_from_atoms(atoms:dict, base_atoms:dict,
return atom[0] return atom[0]
else: # atom with args else: # atom with args
return '{}({})'.format(atom[0], ','.join(map(get_uid_from_atom, atom[1]))) return '{}({})'.format(atom[0], ','.join(map(get_uid_from_atom, atom[1])))
raise ValueError("Malformed node uid found: " + str(atom)) raise ValueError(f"Malformed node uid of type '{type(atom)}' found: {atom}")
for link in get_atoms_of_predicate('link'): for link in get_atoms_of_predicate('link'):
if len(link) == 2: if len(link) == 2:
...@@ -93,22 +93,22 @@ def visual_config_from_atoms(atoms:dict, base_atoms:dict, ...@@ -93,22 +93,22 @@ def visual_config_from_atoms(atoms:dict, base_atoms:dict,
max_label_width[src, trg] = int(value) max_label_width[src, trg] = int(value)
for annotation in get_atoms_of_predicate('annot'): for annotation in get_atoms_of_predicate('annot'):
if len(annotation) == 3: if len(annotation) == 3:
type, node, content = annotation type_, node, content = annotation
node = get_uid_from_atom(node) node = get_uid_from_atom(node)
if type == 'upper': if type_ == 'upper':
upper_annotations[node]['taillabel'].add(content.strip('"')) upper_annotations[node]['taillabel'].add(content.strip('"'))
elif type == 'lower': elif type_ == 'lower':
lower_annotations[node]['headlabel'].add(content.strip('"')) lower_annotations[node]['headlabel'].add(content.strip('"'))
elif type == 'label': elif type_ == 'label':
properties[node]['label'].add(content.strip('"')) properties[node]['label'].add(content.strip('"'))
else: else:
print('Unknow annotation type: {}'.format(type)) print('Unknow annotation type: {}'.format(type_))
elif len(annotation) == 4: # other field elif len(annotation) == 4: # other field
type, node, field, content = annotation type_, node, field, content = annotation
node = get_uid_from_atom(node) node = get_uid_from_atom(node)
if type == 'upper': if type_ == 'upper':
upper_annotations[node][field].add(content.strip('"')) upper_annotations[node][field].add(content.strip('"'))
elif type == 'lower': elif type_ == 'lower':
lower_annotations[node][field].add(content.strip('"')) lower_annotations[node][field].add(content.strip('"'))
for property in get_atoms_of_predicate('dot_property'): for property in get_atoms_of_predicate('dot_property'):
if len(property) == 3: # it's for node if len(property) == 3: # it's for node
......
...@@ -4,9 +4,12 @@ Call example in main. ...@@ -4,9 +4,12 @@ Call example in main.
""" """
import os import os
import time
import json
import clyngor import clyngor
import tempfile import tempfile
from PIL import Image from PIL import Image
from collections import OrderedDict
from . import utils from . import utils
from . import Script from . import Script
from . import asp_to_dot from . import asp_to_dot
...@@ -22,36 +25,85 @@ EXT_TO_TYPE = utils.reverse_dict({ ...@@ -22,36 +25,85 @@ EXT_TO_TYPE = utils.reverse_dict({
LOADABLE = {'Python', 'ASP', 'json/ASP'} LOADABLE = {'Python', 'ASP', 'json/ASP'}
def single_image_from_filenames(fnames:[str], outfile:str=None, dotfile:str=None, return_image:bool=True) -> Image or None: def single_image_from_filenames(fnames:[str], outfile:str=None, dotfile:str=None, return_image:bool=True, verbosity:int=0) -> Image or None:
pipeline = build_pipeline(fnames) pipeline = build_pipeline(fnames, verbosity)
final_context = run(pipeline) final_context = run(pipeline, verbosity=verbosity)
return compile_to_single_image(final_context, outfile=outfile, dotfile=dotfile, return_image=return_image) return compile_to_single_image(final_context, outfile=outfile, dotfile=dotfile, return_image=return_image, verbosity=verbosity)
def build_pipeline(fnames:[str]) -> [Script]: def build_pipeline(fnames:[str], verbosity:int=0) -> [Script]:
"Yield scripts found in given filenames" "Yield scripts found in given filenames"
for fname in fnames: for fname in fnames:
if isinstance(fname, Script):
yield fname
continue
ext = os.path.splitext(fname)[1] ext = os.path.splitext(fname)[1]
ftype = EXT_TO_TYPE.get(ext, 'unknow type') ftype = EXT_TO_TYPE.get(ext, 'unknow type')
if ftype not in LOADABLE: if ftype not in LOADABLE:
raise ValueError(f"The type '{ftype}' can't be loaded") raise ValueError(f"The type '{ftype}' can't be loaded")
yield from module_loader.build_scripts_from_file(fname) yield from module_loader.build_scripts_from_file(fname)
def build_pipeline_from_configfile(config:str, verbosity:int=0) -> [Script]:
with open(config) as fd:
configdata = json.loads(fd.read(), object_pairs_hook=OrderedDict)
yield from build_pipeline_from_json(configdata)
def run(scripts:[Script], initial_context:str='') -> str: def build_pipeline_from_json(jsondata:dict, verbosity:int=0) -> [Script]:
if isinstance(jsondata, list):
for item in jsondata:
yield from build_pipeline_from_json(item)
for name, options in jsondata.items():
scripts = module_loader.build_scripts_from_file(name)
for script in scripts:
script.options_values.update(options)
yield script
build_pipeline.from_configfile = build_pipeline_from_configfile
build_pipeline.from_json = build_pipeline_from_json
def run(scripts:[Script], initial_context:str='', verbosity:int=0) -> str:
if verbosity >= 1:
scripts = tuple(scripts)
print(f"RUNNING {len(scripts)} SCRIPTS…")
run_start = time.time()
context = initial_context context = initial_context
for script in scripts: for idx, script in enumerate(scripts, start=1):
script_start = time.time()
if script.input_mode is str:
input_data = context
else: # need a solving
assert script.input_mode is iter
input_data = solve_context(context)
if verbosity >= 2:
print('RUN run_on…', end='', flush=True)
new_context = script.run_on(input_data, **script.options_values)
if verbosity >= 2:
print('OK!')
# if verbosity >= 3:
# print('NEW CONTEXT:', new_context)
if script.erase_context: if script.erase_context:
context = script.run_on(context) context = new_context
else: else:
context += '\n' + script.run_on(context) context += '\n' + new_context
script_time = round(time.time() - script_start, 2)
if verbosity >= 1:
print(f"SCRIPT {idx}: {script.name} add {len(new_context.splitlines())} lines to the context in {script_time}s.")
run_time = round(time.time() - run_start, 2)
if verbosity >= 1:
print(f"RUN {len(context.splitlines())} lines built in {run_time}s.")
return context return context
def compile_to_single_image(context:str, outfile:str=None, dotfile:str=None, return_image:bool=True) -> Image or None: def solve_context(context:str) -> clyngor.Answers:
return clyngor.solve(inline=context).by_predicate.careful_parsing.int_not_parsed
def compile_to_single_image(context:str, outfile:str=None, dotfile:str=None,
return_image:bool=True, verbosity:int=0) -> Image or None:
"Return a pillow.Image object, or write it to outfile if given" "Return a pillow.Image object, or write it to outfile if given"
configs = asp_to_dot.visual_config_from_asp( configs = asp_to_dot.visual_config_from_asp(
clyngor.solve(inline=context) solve_context(context)
) )
dot = dot_writer.one_graph_from_configs(configs) dot = dot_writer.one_graph_from_configs(configs)
del_outfile = False del_outfile = False
......
# encoding: utf8
"""Routines converting various format into ASP.
"""
import os
import csv
import shutil
import tempfile
import itertools
from biseau import core
def format_from_filename(fname:str) -> str or None:
"""Return the format associated with given filename"""
return os.path.splitext(fname)[1][1:]
def convert(fin:str, fout:str):
"""Convert content found in input file to the format inferred from
given output filename, where it's then wrote.
"""
input_format = format_from_filename(fin)
output_format = format_from_filename(fout)
if output_format != 'lp':
raise NotImplementedError("Not in the scope of this project")
if input_format == output_format:
shutil.copy(fin, fout)
elif input_format not in convert_to_lp:
raise NotImplementedError("Format {} not handled".format(input_format))
else:
convert_to_lp[input_format](fin, fout)
def convert_cxt_to_lp(fin:str, fout:str):
with open(fin) as ifd, open(fout, 'w') as ofd:
assert next(ifd) == 'B\n', 'expects a B'
assert next(ifd) == '\n', 'expects empty line'
nb_obj, nb_att = map(int, (next(ifd), next(ifd)))
assert next(ifd) == '\n', 'expects empty line'
objects = tuple(next(ifd).strip() for _ in range(nb_obj))
attributes = tuple(next(ifd).strip() for _ in range(nb_att))
for object, properties in zip(objects, ifd):
intent = itertools.compress(attributes, (char.lower() == 'x'
for char in properties))
for prop in intent:
ofd.write('rel("{}","{}").'.format(object, prop))
def convert_slf_to_lp(fin:str, fout:str):
with tempfile.NamedTemporaryFile('w', delete=True) as fd:
core.slf_to_cxt.file_to_file(fin, fd.name)
return convert_cxt_to_lp(fd.name, fout)
def convert_txt_to_lp(fin:str, fout:str):
with open(fin) as ifd, open(fout, 'w') as ofd:
lines = csv.reader(ifd, delimiter='|')
attributes = tuple(map(str.strip, next(lines)[1:-1])) # first and last fields are empty
for object, *props in lines:
intent = itertools.compress(attributes, (char.strip().lower() == 'x'
for char in props))
for prop in intent:
ofd.write('rel("{}","{}").'.format(object.strip(), prop))
def convert_csv_to_lp(fin:str, fout:str):
with open(fin) as ifd, open(fout, 'w') as ofd:
lines = csv.reader(ifd, delimiter=',')
attributes = tuple(map(str.strip, next(lines)[1:])) # first field is empty
for object, *props in lines:
intent = itertools.compress(attributes, (char.strip().lower() == 'x'
for char in props))
for prop in intent:
ofd.write('rel("{}","{}").'.format(object.strip(), prop))
convert_to_lp = {
'cxt': convert_cxt_to_lp,
'slf': convert_slf_to_lp,
'txt': convert_txt_to_lp,
'csv': convert_csv_to_lp,
}
...@@ -28,13 +28,14 @@ class ScriptError(ValueError): ...@@ -28,13 +28,14 @@ class ScriptError(ValueError):
pass pass
Script = namedtuple('Script', 'name, tags, description, module, run_on, options, input_mode, incompatible, active_by_default, spec_inputs, spec_outputs, inputs, outputs, source_view, disabled, erase_context') Script = namedtuple('Script', 'name, tags, description, module, run_on, options, options_values, input_mode, incompatible, active_by_default, spec_inputs, spec_outputs, inputs, outputs, source_view, disabled, erase_context')
# name -- human readable name # name -- human readable name
# tags -- set of tags identifying the script # tags -- set of tags identifying the script
# description -- human readable and high level description of the script # description -- human readable and high level description of the script
# module -- reference to the module itself # module -- reference to the module itself
# run_on -- function in module to call on context # run_on -- function in module to call on context
# options -- list of (name, type, default, description) describing each option # options -- list of (name, type, default, description) describing each option
# options_value -- mutable mapping allowing to set options value to be used
# input_mode -- define if run_on must receive the context or the resulting ASP models # input_mode -- define if run_on must receive the context or the resulting ASP models
# incompatible -- list of incompatibles modules # incompatible -- list of incompatibles modules
# active_by_default -- true if the script must be activated at start # active_by_default -- true if the script must be activated at start
...@@ -47,6 +48,7 @@ Script = namedtuple('Script', 'name, tags, description, module, run_on, options, ...@@ -47,6 +48,7 @@ Script = namedtuple('Script', 'name, tags, description, module, run_on, options,
# erase_context -- true if the script erase the context (default: false, context is kept) # erase_context -- true if the script erase the context (default: false, context is kept)
def gen_scripts_in_dir(dirname:str, extensions:[str]=('py', 'lp', 'json'), def gen_scripts_in_dir(dirname:str, extensions:[str]=('py', 'lp', 'json'),
filter_prefixes:[str]='_') -> (str, str): filter_prefixes:[str]='_') -> (str, str):
yield from ( yield from (
...@@ -209,15 +211,16 @@ def build_script_from_json(module_def:dict) -> Script: ...@@ -209,15 +211,16 @@ def build_script_from_json(module_def:dict) -> Script:
def run_on(context:str): def run_on(context:str):
assert isinstance(context, str), (type(context), context) assert isinstance(context, str), (type(context), context)
with open(fname) as fd: with open(fname) as fd:
return context + '\n' + fd.read() return fd.read()
module.editor = gui.UserCodeWidget module_def['erase_context'] = False
elif 'ASP' in module_def: elif 'ASP' in module_def:
asp_code = module_def['ASP'] asp_code = module_def['ASP']
if os.path.exists(asp_code): if os.path.exists(asp_code):
raise ScriptError("JSON script {} put an ASP file ({}) as raw " raise ScriptError("JSON script {} put an ASP file ({}) as raw "
"ASP code.".format(module.NAME, asp_code)) "ASP code.".format(module.NAME, asp_code))
def run_on(context:str): def run_on(context:str):
return context + '\n' + asp_code return asp_code
module_def['erase_context'] = False
module.source_view = asp_code module.source_view = asp_code
elif 'python' in module_def or 'python file' in module_def: elif 'python' in module_def or 'python file' in module_def:
if 'python' in module_def: if 'python' in module_def:
...@@ -243,7 +246,6 @@ def build_script_from_json(module_def:dict) -> Script: ...@@ -243,7 +246,6 @@ def build_script_from_json(module_def:dict) -> Script:
return utils.join_on_genstr(namespace['func'])() return utils.join_on_genstr(namespace['func'])()
except: except:
print('Imported Python error:', traceback.format_exc()) print('Imported Python error:', traceback.format_exc())
module.editor = gui.UserPythonCodeWidget
module.source_view = pycode # just the user written part, not the function encapsulation module.source_view = pycode # just the user written part, not the function encapsulation
else: else:
raise ValueError("JSON script {} do not have any code field ('ASP' " raise ValueError("JSON script {} do not have any code field ('ASP' "
...@@ -338,6 +340,7 @@ def build_script_from_module(module) -> Script or ScriptError: ...@@ -338,6 +340,7 @@ def build_script_from_module(module) -> Script or ScriptError:
module=module, module=module,
run_on=utils.join_on_genstr(getattr(module, 'run_on', None)), run_on=utils.join_on_genstr(getattr(module, 'run_on', None)),
options=tuple(options), options=tuple(options),
options_values={},
input_mode=input_mode, input_mode=input_mode,
incompatible=frozenset(getattr(module, 'INCOMPATIBLE', ())), incompatible=frozenset(getattr(module, 'INCOMPATIBLE', ())),
active_by_default=active_by_default, active_by_default=active_by_default,
......
{
"scripts/context.py": {
"fname": "contexts/human.cxt"
},
"scripts/build_concepts.py": {
"type": "formal",
"careful_parsing": true
},
"scripts/galois-lattice.json": {
},
"scripts/show_galois_lattice.py": {
"dpi": 300,
"use_aoc_as_label": true
}
}
{
"scripts/open_data.py": {
"file": "scripts/example.lp"
},
"scripts/ASPExtractor.py": {
"shows": "#show link/2.",
"export_file": "extracted.lp"
}
}
Contexts here comes from [concepts lib](https://github.com/xflr6/concepts/tree/master/examples),
which probably took them from another source.
/home/lbourneu/data/mushrooms/agaricus-lepiota.lp
\ No newline at end of file
product("d","R2").
dreaction("R_importS2").
reversible("R_importS2").
reactant("c","R5").
reactant("f","R_exportF").
product("S1","R_importS1").
dreaction("R2").
product("S2","R_importS2").
dreaction("R1").
reactant("S3","R1").
reactant("b","R0").
dreaction("R0").
dreaction("R4").
reactant("S2","R2").
reactant("e","R4").
dreaction("R_importS1").
product("S3","R0").
reactant("c","R2").
dreaction("R5").
product("f","R5").
dreaction("R_exportF").
reversible("R_importS1").
product("c","R4").
reactant("a","R5").
reactant("d","R3").
product("b","R1").
product("e","R3").
dreaction("R3").
rel(x,yx).
rel(y,yx).
rel(x,xz).
rel(z,xz).
rel(z,yz).
rel(y,yz).
rel(x,wx).
rel(w,wx).
rel(y,wy).
rel(w,wy).
rel(z,wz).
rel(w,wz).
rel(x,yx).
rel(y,yx).
rel(x,xz).
rel(z,xz).
rel(z,yz).
rel(y,yz).
rel(wy,wyx).
rel(yx,wyx).
rel(wx,wyx).
rel(xz,wxz).
rel(wx,wxz).
rel(wz,wxz).
rel(wy,wyz).
rel(yz,wyz).
rel(wz,wyz).
rel(xz,yxz).
rel(yx,yxz).
rel(yz,yxz).
taller(alex,brian).
taller(charlie,alex).
taller(daniel,edward).
taller(alex,daniel).
B
10
7
0
1
2
3
4
5
6
7
8
9
a
b
c
d
e
f
g
X.XXXXX
.....XX
XXX.XX.
XXX..XX
.X.X.XX
XXXX..X
.XXXX.X
X....XX
XXXXXXX
XX.X.XX
% Data from
% https://medium.com/@nykolas.z/dns-resolvers-performance-compared-cloudflare-x-google-x-quad9-x-opendns-149e803734e5
% Features: privacy DNSCrypt DNSoverHTTPS DNSoverTLS
% Google X - X X
% CloudFlare X - X X
% Quad9 X - - X
% OpenDNS - X - X
% Norton - - - -