Commit 028ae071 authored by Ryan Herbert's avatar Ryan Herbert Committed by Mikael Salson

add some basic prototypes for rapidly parsing json files

parent 3245fcb3
#!/usr/bin/python
import ijson
fusedfile = open('/home/ryan/fused_file.fused', 'r')
'''
objects = ijson.items(fusedfile, 'clones.item')
affectSigns = (c['seg']['affectSigns'] for c in objects)
count = 1
for afs in affectSigns:
print('affect: {}, {}'.format(count, afs))
count = count + 1
'''
parser = ijson.parse(fusedfile)
def pretty_print(prefix, event, value, previous_event):
end = None
if event == 'start_map':
mstr = '{{'
elif event == 'end_map':
mstr = '}},'
elif event == 'start_array':
mstr = '['
elif event == 'end_array':
mstr = '],'
elif event == 'map_key':
mstr = '\'{}\':'
end = ''
elif event == 'string':
mstr = '\'{}\','
else:
mstr = '{},'
padding = ''
if previous_event != 'map_key':
if len(prefix) > 0:
padding = ''.join(['\t' for i in range(len(prefix.split('.')))])
mstr = '{}' + mstr
print(mstr.format(padding, value), end=end)
previous = ''
for prefix, event, value in parser:
prefixes = ['clones.item.seg.affectSigns', 'clones.item.seg.affectValues', 'similarity.item']
#There must be a better way !!!
cond = any(prefix.startswith(item) for item in prefixes) \
or (any(item.startswith(prefix) for item in prefixes) \
and (value is None or any(item.startswith(prefix + '.' + str(value)) for item in prefixes) \
or any(item.startswith(str(value)) for item in prefixes)))
if cond:
pretty_print(prefix, event, value, previous)
previous = event
#!/usr/bin/python
from jsonstreamer import JSONStreamer
def _catch_all(event_name, *args):
print('\t{} : {}'.format(event_name, args))
class VidjilStreamer(JSONStreamer):
def __init__(self):
super(VidjilStreamer, self).__init__()
def value_listener(self, value, *args):
print('stack: {}'.format(self._stack))
print('\tValue: {}'.format(value))
def run(self):
fusefile = open('/home/ryan/fused_file.fused', 'r')
print("parsing the fuse:")
streamer.add_catch_all_listener(_catch_all)
#self.add_listener(JSONStreamer.VALUE_EVENT, self.value_listener)
self.consume(fusefile.read())
streamer = VidjilStreamer()
streamer.run()
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment