Commit 71cf75ec authored by Ryan Herbert's avatar Ryan Herbert Committed by Mikaël Salson
Browse files use a faster parser

use yajl2 as the ijson parser since it is much faster.

See #3234,#3235
parent 0c0ca52f
import ijson
import ijson.backends.yajl2 as ijson
from six import string_types
from enum import Enum
......@@ -19,7 +19,7 @@ class VidjilWriter(object):
def __enter__(self):
if self._filepath:
self.file = open(self._filepath, 'w')
self.file = open(self._filepath, 'wb')
return self
def __exit__(self, exc_type, exc_value, traceback):
......@@ -122,14 +122,14 @@ class VidjilParser(object):
self.prefixes = []
def initModel(self, model_path):
with open(model_path, 'r') as model:
with open(model_path, 'rb') as model:
parser = ijson.parse(model)
for prefix, event, value in parser:
if (prefix, event) not in self._model_prefixes:
self._model_prefixes.append((prefix, event))
def validate(self, filepath):
with open(filepath, 'r') as vfile:
with open(filepath, 'rb') as vfile:
parser = ijson.parse(vfile)
model = list(self._model_prefixes)
for prefix, event, value in parser:
......@@ -151,7 +151,7 @@ class VidjilParser(object):
def extract(self, filepath):
vidjilfile = open(filepath, 'r')
vidjilfile = open(filepath, 'rb')
parser = ijson.parse(vidjilfile)
with self.writer() as writer:
return self._extract(parser, writer)
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment