Commit 66cc5287 authored by Robin Tissot's avatar Robin Tissot
Browse files

Fixes fetching the entire db in memory for a migration.

parent 8787766b
...@@ -4,17 +4,37 @@ import uuid ...@@ -4,17 +4,37 @@ import uuid
from django.db import migrations from django.db import migrations
def batch_qs(qs, batch_size=1000):
"""
Returns a (start, end, total, queryset) tuple for each batch in the given
queryset.
Usage:
# Make sure to order your querset
article_qs = Article.objects.order_by('id')
for start, end, total, qs in batch_qs(article_qs):
print "Now processing %s - %s of %s" % (start + 1, end, total)
for article in qs:
print article.body
"""
total = qs.count()
for start in range(0, total, batch_size):
end = min(start + batch_size, total)
yield (start, end, total, qs[start:end])
def forward(apps, se): def forward(apps, se):
Block = apps.get_model('core', 'Block') Block = apps.get_model('core', 'Block')
Line = apps.get_model('core', 'Line') Line = apps.get_model('core', 'Line')
for block in Block.objects.filter(external_id=None): for s,e,t, blocks in batch_qs(Block.objects.filter(external_id=None)):
block.external_id = 'eSc_textblock_%s' % str(uuid.uuid4())[:8] for block in blocks:
block.save() block.external_id = 'eSc_textblock_%s' % str(uuid.uuid4())[:8]
block.save()
for line in Line.objects.filter(external_id=None): for s,e,t, lines in batch_qs(Line.objects.filter(external_id=None)):
line.external_id = 'eSc_line_%s' % str(uuid.uuid4())[:8] for line in lines:
line.save() line.external_id = 'eSc_line_%s' % str(uuid.uuid4())[:8]
line.save()
def backward(apps, se): def backward(apps, se):
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment