rename csquares based detection

parent 1453837a
...@@ -9,17 +9,16 @@ import functools ...@@ -9,17 +9,16 @@ import functools
import concurrent.futures import concurrent.futures
import glob import glob
import logging import logging
import math
import pickle import pickle
import sys
from sesamelib.geometry import IndexedGeometry, distance from sesamelib.geometry import IndexedGeometry, distance
from sesamelib.faust import BaseDynamicMessage from sesamelib.faust import BaseDynamicMessage
import faust import faust
logging.basicConfig(level=logging.INFO)
APP_NAME = "ca1-015-csquare" APP_NAME = "ca1-015-csquares-%s"
if __name__ == "__main__": if __name__ == "__main__":
parser = argparse.ArgumentParser("Mutate the ais dynamic signal") parser = argparse.ArgumentParser("Mutate the ais dynamic signal")
...@@ -27,9 +26,14 @@ if __name__ == "__main__": ...@@ -27,9 +26,14 @@ if __name__ == "__main__":
help="Kafka bootstrap servers", help="Kafka bootstrap servers",
action="append", action="append",
required=True) required=True)
parser.add_argument("-c", "--cell_size", help="cell_size to use. Will look in cell_size subdirectory to load the geometries",
type=float, default=0.0)
args = parser.parse_args() args = parser.parse_args()
files = glob.glob("data/*.pickle") logging.info(args)
files = glob.glob("/data/%s/*.pickle" % args.cell_size)
# will hold all the indexed geometries # will hold all the indexed geometries
geometries = [] geometries = []
for f in files: for f in files:
...@@ -37,12 +41,14 @@ if __name__ == "__main__": ...@@ -37,12 +41,14 @@ if __name__ == "__main__":
geometry = pickle.load(f) geometry = pickle.load(f)
geometries.append(geometry) geometries.append(geometry)
logging.info("Loaded %s geometries" % len(geometries))
loop = asyncio.get_event_loop() loop = asyncio.get_event_loop()
pool = concurrent.futures.ThreadPoolExecutor() pool = concurrent.futures.ThreadPoolExecutor()
app = faust.App( app = faust.App(
APP_NAME, APP_NAME % args.cell_size,
loop=loop, loop=loop,
broker="kafka://{}".format(",".join(args.bootstrap_servers)) broker="kafka://{}".format(",".join(args.bootstrap_servers))
) )
...@@ -50,7 +56,7 @@ if __name__ == "__main__": ...@@ -50,7 +56,7 @@ if __name__ == "__main__":
topic = app.topic("ais.dynamic", topic = app.topic("ais.dynamic",
key_type=str, key_type=str,
value_type=BaseDynamicMessage) value_type=BaseDynamicMessage)
out_topic = app.topic(APP_NAME) out_topic = app.topic(APP_NAME % args.cell_size)
pmsgs = {} pmsgs = {}
...@@ -67,6 +73,7 @@ if __name__ == "__main__": ...@@ -67,6 +73,7 @@ if __name__ == "__main__":
y, y,
geometry)) geometry))
if d <= 0.0: if d <= 0.0:
print(msg)
await out_topic.send(key=str(msg.mmsi), value=msg) await out_topic.send(key=str(msg.mmsi), value=msg)
app.finalize() app.finalize()
......
...@@ -33,7 +33,7 @@ services: ...@@ -33,7 +33,7 @@ services:
image: $NAMESPACE/ingestion:$TARGET_REF image: $NAMESPACE/ingestion:$TARGET_REF
command: --bootstrap_servers kafka:9092 "/bigdata/groups/sesame/orbcomm/aivdm/2017/03/01/*.nm4" command: --bootstrap_servers kafka:9092 "/bigdata/groups/sesame/orbcomm/aivdm/2017/03/01/*.nm4"
container_name: ingestion container_name: ingestion
# don't restart otherwise messages will be indexed several times... # don':t restart otherwise messages will be indexed several times...
restart: on-failure restart: on-failure
build: build:
context: ingestion context: ingestion
...@@ -41,7 +41,7 @@ services: ...@@ -41,7 +41,7 @@ services:
TARGET_REF: $TARGET_REF TARGET_REF: $TARGET_REF
NAMESPACE: $NAMESPACE NAMESPACE: $NAMESPACE
volumes: volumes:
- ../bigdata:/bigdata - ../bigdata/:/bigdata
depends_on: depends_on:
- base_gdal - base_gdal
...@@ -88,16 +88,18 @@ services: ...@@ -88,16 +88,18 @@ services:
depends_on: depends_on:
- base_gdal - base_gdal
ca1-015-csquare: ca1-015-csquares:
image: $NAMESPACE/ca1-015-csquare:$TARGET_REF image: $NAMESPACE/ca1-015-csquares:$TARGET_REF
command: --bootstrap_servers kafka:9092 command: --bootstrap_servers kafka:9092 --cell_size 0.25
container_name: ca1-015-csquare container_name: ca1-015-csquares
restart: always restart: always
build: build:
context: ca1-015-csquare context: ca1-015-csquares
args: args:
TARGET_REF: $TARGET_REF TARGET_REF: $TARGET_REF
NAMESPACE: $NAMESPACE NAMESPACE: $NAMESPACE
volumes:
- ../bigdata/groups/sesame/csquares:/data
depends_on: depends_on:
- base_gdal - base_gdal
############################ ############################
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment