Skip to content
GitLab
Projects
Groups
Snippets
/
Help
Help
Support
Community forum
Keyboard shortcuts
?
Submit feedback
Contribute to GitLab
Sign in
Toggle navigation
Menu
Open sidebar
vidjil
vidjil
Commits
028ae071
Commit
028ae071
authored
May 22, 2018
by
Ryan Herbert
Committed by
Mikaël Salson
Jul 18, 2018
Browse files
add some basic prototypes for rapidly parsing json files
parent
3245fcb3
Changes
2
Hide whitespace changes
Inline
Side-by-side
ijson-test.py
0 → 100644
View file @
028ae071
#!/usr/bin/python
import
ijson
fusedfile
=
open
(
'/home/ryan/fused_file.fused'
,
'r'
)
'''
objects = ijson.items(fusedfile, 'clones.item')
affectSigns = (c['seg']['affectSigns'] for c in objects)
count = 1
for afs in affectSigns:
print('affect: {}, {}'.format(count, afs))
count = count + 1
'''
parser
=
ijson
.
parse
(
fusedfile
)
def
pretty_print
(
prefix
,
event
,
value
,
previous_event
):
end
=
None
if
event
==
'start_map'
:
mstr
=
'{{'
elif
event
==
'end_map'
:
mstr
=
'}},'
elif
event
==
'start_array'
:
mstr
=
'['
elif
event
==
'end_array'
:
mstr
=
'],'
elif
event
==
'map_key'
:
mstr
=
'
\'
{}
\'
:'
end
=
''
elif
event
==
'string'
:
mstr
=
'
\'
{}
\'
,'
else
:
mstr
=
'{},'
padding
=
''
if
previous_event
!=
'map_key'
:
if
len
(
prefix
)
>
0
:
padding
=
''
.
join
([
'
\t
'
for
i
in
range
(
len
(
prefix
.
split
(
'.'
)))])
mstr
=
'{}'
+
mstr
print
(
mstr
.
format
(
padding
,
value
),
end
=
end
)
previous
=
''
for
prefix
,
event
,
value
in
parser
:
prefixes
=
[
'clones.item.seg.affectSigns'
,
'clones.item.seg.affectValues'
,
'similarity.item'
]
#There must be a better way !!!
cond
=
any
(
prefix
.
startswith
(
item
)
for
item
in
prefixes
)
\
or
(
any
(
item
.
startswith
(
prefix
)
for
item
in
prefixes
)
\
and
(
value
is
None
or
any
(
item
.
startswith
(
prefix
+
'.'
+
str
(
value
))
for
item
in
prefixes
)
\
or
any
(
item
.
startswith
(
str
(
value
))
for
item
in
prefixes
)))
if
cond
:
pretty_print
(
prefix
,
event
,
value
,
previous
)
previous
=
event
jsonstreamer-test.py
0 → 100644
View file @
028ae071
#!/usr/bin/python
from
jsonstreamer
import
JSONStreamer
def
_catch_all
(
event_name
,
*
args
):
print
(
'
\t
{} : {}'
.
format
(
event_name
,
args
))
class
VidjilStreamer
(
JSONStreamer
):
def
__init__
(
self
):
super
(
VidjilStreamer
,
self
).
__init__
()
def
value_listener
(
self
,
value
,
*
args
):
print
(
'stack: {}'
.
format
(
self
.
_stack
))
print
(
'
\t
Value: {}'
.
format
(
value
))
def
run
(
self
):
fusefile
=
open
(
'/home/ryan/fused_file.fused'
,
'r'
)
print
(
"parsing the fuse:"
)
streamer
.
add_catch_all_listener
(
_catch_all
)
#self.add_listener(JSONStreamer.VALUE_EVENT, self.value_listener)
self
.
consume
(
fusefile
.
read
())
streamer
=
VidjilStreamer
()
streamer
.
run
()
Write
Preview
Supports
Markdown
0%
Try again
or
attach a new file
.
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment