Commit 6620027b authored by Deniro StopCovid's avatar Deniro StopCovid

feat: Use the property var robert.server.request-time-delta-tolerance mapped...

feat: Use the property var robert.server.request-time-delta-tolerance mapped on the env var ROBERT_SERVER_REQUEST_TIME_DELTA_TOLERANCE
parent 79b8025c
......@@ -19,7 +19,7 @@
<groupId>fr.gouv.stopc</groupId>
<artifactId>robert-server</artifactId>
<version>1.4.0-SNAPSHOT</version>
<version>1.5.0-SNAPSHOT</version>
<name>robert-server</name>
<packaging>pom</packaging>
<description>Projet principal</description>
......
......@@ -13,7 +13,7 @@
<parent>
<groupId>fr.gouv.stopc</groupId>
<artifactId>robert-server</artifactId>
<version>1.4.0-SNAPSHOT</version>
<version>1.5.0-SNAPSHOT</version>
</parent>
<artifactId>robert-crypto-grpc-server-messaging</artifactId>
......
// Copyright 2015 The gRPC Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
......@@ -47,6 +46,11 @@ message HSMCacheStatusResponse {
repeated string aliases = 1;
}
message ErrorMessage {
int32 code = 1;
string description = 2;
}
message CreateRegistrationRequest {
bytes clientPublicKey = 1;
int32 fromEpochId = 2;
......@@ -57,6 +61,7 @@ message CreateRegistrationRequest {
message CreateRegistrationResponse {
bytes idA = 1;
bytes tuples = 2;
ErrorMessage error = 3;
}
message GetIdFromAuthRequest {
......@@ -70,6 +75,7 @@ message GetIdFromAuthRequest {
message GetIdFromAuthResponse {
bytes idA = 1;
int32 epochId = 2;
ErrorMessage error = 3;
}
message GetIdFromStatusRequest {
......@@ -86,6 +92,7 @@ message GetIdFromStatusResponse {
bytes idA = 1;
int32 epochId = 2;
bytes tuples = 3;
ErrorMessage error = 4;
}
message GetInfoFromHelloMessageRequest {
......@@ -94,12 +101,14 @@ message GetInfoFromHelloMessageRequest {
bytes mac = 3;
int32 timeSent = 4; // time hello message was sent from emitting device (unsigned 16 bits)
int64 timeReceived = 5; // time hello message was received on device (unsigned 32 bits, stored in long)
bytes serverCountryCode = 6;
}
message GetInfoFromHelloMessageResponse {
bytes countryCode = 1;
bytes idA = 2;
int32 epochId = 3;
ErrorMessage error = 4;
}
message DeleteIdRequest {
......@@ -111,4 +120,5 @@ message DeleteIdRequest {
message DeleteIdResponse {
bytes idA = 1;
}
\ No newline at end of file
ErrorMessage error = 2;
}
......@@ -13,7 +13,7 @@
<parent>
<groupId>fr.gouv.stopc</groupId>
<artifactId>robert-server</artifactId>
<version>1.4.0-SNAPSHOT</version>
<version>1.5.0-SNAPSHOT</version>
</parent>
......
......@@ -16,7 +16,13 @@ import java.security.cert.CertificateException;
import java.time.LocalDate;
import java.time.ZoneId;
import java.time.format.DateTimeFormatter;
import java.util.*;
import java.util.HashMap;
import java.util.Map;
import java.util.List;
import java.util.ArrayList;
import java.util.Objects;
import java.util.Optional;
import java.util.Comparator;
import java.util.stream.Collectors;
import javax.crypto.spec.SecretKeySpec;
......
......@@ -13,7 +13,7 @@
<parent>
<groupId>fr.gouv.stopc</groupId>
<artifactId>robert-server</artifactId>
<version>1.4.0-SNAPSHOT</version>
<version>1.5.0-SNAPSHOT</version>
</parent>
<artifactId>robert-crypto-grpc-server</artifactId>
......
......@@ -4,7 +4,6 @@ import java.io.IOException;
import javax.inject.Inject;
import fr.gouv.stopc.robert.server.common.utils.ByteUtils;
import org.springframework.context.annotation.Configuration;
import fr.gouv.stopc.robert.crypto.grpc.server.storage.cryptographic.service.ICryptographicStorageService;
......@@ -19,14 +18,9 @@ public class CryptoServiceConfiguration {
PropertyLoader propertyLoader,
ICryptographicStorageService cryptoStorageService) throws IOException, InterruptedException {
// Init the cryptographic Storage
// Init the cryptographic Storage
cryptoStorageService.init(propertyLoader.getKeyStorePassword(), propertyLoader.getKeyStoreConfigFile());
// Store if does not exist the public and key
//cryptoStorageService.addECDHKeys(propertyLoader.getServerPublicKey(), propertyLoader.getServerPrivateKey());
//cryptoStorageService.addKekKeysIfNotExist(
// ByteUtils.generateRandom(32),
// ByteUtils.generateRandom(32));
server.initPort(Integer.parseInt(propertyLoader.getCryptoServerPort()));
server.start();
server.blockUntilShutdown();
......
......@@ -5,6 +5,7 @@ import java.util.concurrent.TimeUnit;
import javax.inject.Inject;
import fr.gouv.stopc.robert.crypto.grpc.server.exception.RobertServerCryptoExceptionHandler;
import org.springframework.stereotype.Service;
import fr.gouv.stopc.robert.crypto.grpc.server.messaging.CryptoGrpcServiceImplGrpc.CryptoGrpcServiceImplImplBase;
......@@ -21,12 +22,10 @@ public class CryptoServiceGrpcServer {
private Server server;
private CryptoGrpcServiceImplImplBase cryptoService;
//private HealthGrpc.HealthImplBase healthService;
@Inject
public CryptoServiceGrpcServer(final CryptoGrpcServiceImplImplBase cryptoService) {
this.cryptoService = cryptoService;
//this.healthService = healthService;
}
public CryptoServiceGrpcServer(int port) {
......@@ -36,7 +35,7 @@ public class CryptoServiceGrpcServer {
public CryptoServiceGrpcServer(ServerBuilder<?> serverBuilder, int port) {
this.server = serverBuilder
.addService(cryptoService)
//.addService(healthService)
.intercept(new RobertServerCryptoExceptionHandler())
.build();
this.port = port;
}
......@@ -44,6 +43,7 @@ public class CryptoServiceGrpcServer {
public CryptoServiceGrpcServer(ServerBuilder<?> serverBuilder, int port, BindableService cryptoService) {
this.server = serverBuilder
.addService(cryptoService)
.intercept(new RobertServerCryptoExceptionHandler())
.build();
this.port = port;
......@@ -54,7 +54,7 @@ public class CryptoServiceGrpcServer {
this.server = ServerBuilder
.forPort(port)
.addService(cryptoService)
//.addService(healthService)
.intercept(new RobertServerCryptoExceptionHandler())
.build();
}
public void start() throws IOException {
......
package fr.gouv.stopc.robert.crypto.grpc.server.exception;
import io.grpc.ForwardingServerCallListener;
import io.grpc.Metadata;
import io.grpc.ServerCall;
import io.grpc.ServerCallHandler;
import io.grpc.ServerInterceptor;
import io.grpc.Status;
public class RobertServerCryptoExceptionHandler implements ServerInterceptor {
@Override
public <ReqT, RespT> ServerCall.Listener<ReqT> interceptCall(ServerCall<ReqT, RespT> serverCall, Metadata metadata,
ServerCallHandler<ReqT, RespT> serverCallHandler) {
ServerCall.Listener<ReqT> listener = serverCallHandler.startCall(serverCall, metadata);
return new ExceptionHandlingServerCallListener<>(listener, serverCall, metadata);
}
private class ExceptionHandlingServerCallListener<ReqT, RespT>
extends ForwardingServerCallListener.SimpleForwardingServerCallListener<ReqT> {
private ServerCall<ReqT, RespT> serverCall;
private Metadata metadata;
ExceptionHandlingServerCallListener(ServerCall.Listener<ReqT> listener, ServerCall<ReqT, RespT> serverCall,
Metadata metadata) {
super(listener);
this.serverCall = serverCall;
this.metadata = metadata;
}
@Override
public void onHalfClose() {
try {
super.onHalfClose();
} catch (RuntimeException ex) {
handleException(ex, serverCall, metadata);
throw ex;
}
}
@Override
public void onReady() {
try {
super.onReady();
} catch (RuntimeException ex) {
handleException(ex, serverCall, metadata);
throw ex;
}
}
private void handleException(RuntimeException exception, ServerCall<ReqT, RespT> serverCall, Metadata metadata) {
serverCall.close(Status.UNKNOWN, metadata);
}
}
}
......@@ -9,5 +9,4 @@ public interface ICryptoServerConfigurationService {
* not tied to an instance)
*/
long getServiceTimeStart();
}
package fr.gouv.stopc.robert.crypto.grpc.server.service.impl;
import java.security.*;
import java.security.InvalidKeyException;
import java.security.KeyFactory;
import java.security.KeyPair;
import java.security.NoSuchAlgorithmException;
import java.security.PrivateKey;
import java.security.ProviderException;
import java.security.PublicKey;
import java.security.spec.InvalidKeySpecException;
import java.security.spec.X509EncodedKeySpec;
import java.util.Objects;
......@@ -9,16 +15,13 @@ import java.util.Optional;
import javax.crypto.KeyAgreement;
import javax.inject.Inject;
import org.springframework.stereotype.Service;
import fr.gouv.stopc.robert.crypto.grpc.server.service.IECDHKeyService;
import fr.gouv.stopc.robert.crypto.grpc.server.storage.cryptographic.service.ICryptographicStorageService;
import fr.gouv.stopc.robert.crypto.grpc.server.storage.model.ClientIdentifierBundle;
import fr.gouv.stopc.robert.server.crypto.exception.RobertServerCryptoException;
import fr.gouv.stopc.robert.server.crypto.structure.impl.CryptoHMACSHA256;
import lombok.AllArgsConstructor;
import lombok.Builder;
import lombok.Getter;
import org.springframework.stereotype.Service;
import fr.gouv.stopc.robert.crypto.grpc.server.service.IECDHKeyService;
import lombok.extern.slf4j.Slf4j;
@Slf4j
......@@ -65,7 +68,7 @@ public class ECDHKeyServiceImpl implements IECDHKeyService {
keyAgreement.doPhase(clientPublicKeyAsKey, true);
return keyAgreement.generateSecret();
} catch (NoSuchAlgorithmException | InvalidKeySpecException
| InvalidKeyException | IllegalStateException e) {
| InvalidKeyException | IllegalStateException | ProviderException e) {
log.error("Unable to generate ECDH Keys due to {}", e.getMessage());
}
......
......@@ -13,7 +13,7 @@
<parent>
<groupId>fr.gouv.stopc</groupId>
<artifactId>robert-server</artifactId>
<version>1.4.0-SNAPSHOT</version>
<version>1.5.0-SNAPSHOT</version>
</parent>
<artifactId>robert-server-batch</artifactId>
......@@ -93,13 +93,13 @@
<scope>test</scope>
</dependency>
<!-- Micrometer core dependency -->
<!-- Micrometer core dependency -->
<dependency>
<groupId>io.micrometer</groupId>
<artifactId>micrometer-core</artifactId>
</dependency>
<!-- Micrometer Prometheus registry -->
<!-- Micrometer Prometheus registry -->
<dependency>
<groupId>io.micrometer</groupId>
<artifactId>micrometer-registry-prometheus</artifactId>
......
package fr.gouv.stopc.robert.server.batch.configuration;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import javax.inject.Inject;
import fr.gouv.stopc.robert.server.batch.processor.RegistrationProcessor;
import fr.gouv.stopc.robertserver.database.model.Registration;
import lombok.extern.slf4j.Slf4j;
import org.springframework.batch.core.Job;
import org.springframework.batch.core.Step;
import org.springframework.batch.core.configuration.annotation.EnableBatchProcessing;
import org.springframework.batch.core.configuration.annotation.JobBuilderFactory;
import org.springframework.batch.core.configuration.annotation.StepBuilderFactory;
import org.springframework.batch.core.configuration.annotation.StepScope;
import org.springframework.batch.core.partition.PartitionHandler;
import org.springframework.batch.core.partition.support.Partitioner;
import org.springframework.batch.core.partition.support.TaskExecutorPartitionHandler;
import org.springframework.batch.item.ItemProcessor;
import org.springframework.batch.item.ItemWriter;
import org.springframework.batch.item.data.MongoItemReader;
import org.springframework.batch.item.data.MongoItemWriter;
import org.springframework.batch.item.data.builder.MongoItemWriterBuilder;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
import org.springframework.core.task.SimpleAsyncTaskExecutor;
import org.springframework.core.task.TaskExecutor;
import org.springframework.data.domain.Sort;
import org.springframework.data.domain.Sort.Direction;
import org.springframework.data.mongodb.core.MongoTemplate;
import org.springframework.data.mongodb.core.query.Criteria;
import org.springframework.data.mongodb.core.query.Query;
import fr.gouv.stopc.robert.crypto.grpc.server.client.service.ICryptoServerGrpcClient;
import fr.gouv.stopc.robert.server.batch.listener.ProcessingJobExecutionListener;
import fr.gouv.stopc.robert.server.batch.model.ItemIdMapping;
import fr.gouv.stopc.robert.server.batch.partitioner.RangePartitioner;
import fr.gouv.stopc.robert.server.batch.processor.ContactIdMappingProcessor;
import fr.gouv.stopc.robert.server.batch.processor.ContactProcessor;
import fr.gouv.stopc.robert.server.batch.processor.PurgeOldEpochExpositionsProcessor;
import fr.gouv.stopc.robert.server.batch.processor.RegistrationIdMappingProcessor;
import fr.gouv.stopc.robert.server.batch.processor.RegistrationProcessor;
import fr.gouv.stopc.robert.server.batch.service.ItemIdMappingService;
import fr.gouv.stopc.robert.server.batch.service.ScoringStrategyService;
import fr.gouv.stopc.robert.server.batch.utils.PropertyLoader;
import fr.gouv.stopc.robert.server.batch.writer.ContactItemWriter;
import fr.gouv.stopc.robert.server.batch.writer.RegistrationItemWriter;
import fr.gouv.stopc.robert.server.common.service.IServerConfigurationService;
import fr.gouv.stopc.robert.server.common.utils.TimeUtils;
import fr.gouv.stopc.robertserver.database.model.Contact;
import fr.gouv.stopc.robertserver.database.model.Registration;
import fr.gouv.stopc.robertserver.database.service.ContactService;
import fr.gouv.stopc.robertserver.database.service.IRegistrationService;
import lombok.extern.slf4j.Slf4j;
@Slf4j
@Configuration
@EnableBatchProcessing
public class ContactsProcessingConfiguration {
public static final int GRID_SIZE = 10;
public static final String TOTAL_CONTACT_COUNT_KEY = "totalContactCount";
public static final String TOTAL_REGISTRATION_COUNT_KEY = "totalRegistrationCount";
public static final String TOTAL_REGISTRATION_FOR_PURGE_COUNT_KEY = "totalRegistrationForPurgeCount";
private final IServerConfigurationService serverConfigurationService;
......@@ -45,9 +71,12 @@ public class ContactsProcessingConfiguration {
private final ScoringStrategyService scoringStrategyService;
private final ItemIdMappingService itemIdMappingService;
private final ICryptoServerGrpcClient cryptoServerClient;
private final int CHUNK_SIZE = 10000;
private final int POPULATE_STEP_CHUNK_SIZE = 200000;
private final PropertyLoader propertyLoader;
......@@ -63,6 +92,7 @@ public class ContactsProcessingConfiguration {
final ContactService contactService,
final ICryptoServerGrpcClient cryptoServerClient,
final ScoringStrategyService scoringStrategyService,
final ItemIdMappingService itemIdMappingService,
final PropertyLoader propertyLoader,
final JobBuilderFactory jobBuilderFactory,
final StepBuilderFactory stepBuilderFactory
......@@ -73,13 +103,16 @@ public class ContactsProcessingConfiguration {
this.contactService = contactService;
this.cryptoServerClient = cryptoServerClient;
this.scoringStrategyService = scoringStrategyService;
this.itemIdMappingService = itemIdMappingService;
this.propertyLoader = propertyLoader;
this.stepBuilderFactory = stepBuilderFactory;
this.jobBuilderFactory = jobBuilderFactory;
}
@Bean
public Job scoreAndProcessRisks(Step stepContact, Step stepRegistration) {
public Job scoreAndProcessRisks(Step contactProcessingStep, Step processRegistrationStep, Step purgeOldEpochExpositionsStep,
Step populateRegistrationIdMappingStep, Step populateContactIdMappingStep,
Step populateRegistrationIdMappingForEpochPurgeStep) {
BatchMode batchMode;
......@@ -92,111 +125,320 @@ public class ContactsProcessingConfiguration {
if (batchMode == BatchMode.FULL_REGISTRATION_SCAN_COMPUTE_RISK) {
log.info("Launching registration batch (No contact scoring, risk computation)");
return this.jobBuilderFactory.get("processRegistration").flow(stepRegistration).end().build();
return this.jobBuilderFactory.get("processRegistration")
.listener(new ProcessingJobExecutionListener(TOTAL_REGISTRATION_COUNT_KEY,
registrationService, contactService, serverConfigurationService,
propertyLoader, itemIdMappingService))
.start(populateRegistrationIdMappingStep)
.next(processRegistrationStep).build();
} else if (batchMode == BatchMode.SCORE_CONTACTS_AND_COMPUTE_RISK) {
log.info("Launching contact batch (Contact scoring, Risk computation)");
return this.jobBuilderFactory.get("processContacts").flow(stepContact).end().build();
return this.jobBuilderFactory.get("processContacts")
.listener(new ProcessingJobExecutionListener(TOTAL_CONTACT_COUNT_KEY,
registrationService, contactService, serverConfigurationService,
propertyLoader, itemIdMappingService))
.start(populateContactIdMappingStep)
.next(contactProcessingStep).build();
} else if (batchMode == BatchMode.PURGE_OLD_EPOCH_EXPOSITIONS) {
log.info("Launching purge old epoch exposition batch");
return this.jobBuilderFactory.get("purgeOldEpochExpositions")
.listener(new ProcessingJobExecutionListener(TOTAL_REGISTRATION_FOR_PURGE_COUNT_KEY,
registrationService, contactService, serverConfigurationService,
propertyLoader, itemIdMappingService))
.start(populateRegistrationIdMappingForEpochPurgeStep)
.next(purgeOldEpochExpositionsStep).build();
}
return null;
}
@Bean
public Step stepContact(
MongoItemReader<Contact> mongoContactItemReader,
MongoItemWriter<Contact> mongoContactItemWriter) {
public Step populateRegistrationIdMappingStep(
MongoItemReader<Registration> mongoRegistrationIdMappingItemReader,
MongoItemWriter<ItemIdMapping> mongoRegistrationIdMappingItemWriter) {
return this.stepBuilderFactory.get("readContacts").<Contact, Contact>chunk(CHUNK_SIZE).reader(mongoContactItemReader)
.processor(contactsProcessor()).writer(mongoContactItemWriter).build();
return this.stepBuilderFactory.get("populateRegistrationIdMapping").<Registration, ItemIdMapping>chunk(POPULATE_STEP_CHUNK_SIZE).reader(mongoRegistrationIdMappingItemReader)
.processor(registrationIdMappingProcessor()).writer(mongoRegistrationIdMappingItemWriter).build();
}
@Bean
public Step stepRegistration(
MongoItemReader<Registration> mongoRegistrationItemReader,
MongoItemWriter<Registration> mongoRegistrationItemWriter) {
public Step populateContactIdMappingStep(
MongoItemReader<Contact> mongoContactIdMappingItemReader,
MongoItemWriter<ItemIdMapping> mongoContactIdMappingItemWriter) {
return this.stepBuilderFactory.get("readRegistrations").<Registration, Registration>chunk(CHUNK_SIZE).reader(mongoRegistrationItemReader)
.processor(registrationsProcessor()).writer(mongoRegistrationItemWriter).build();
return this.stepBuilderFactory.get("populateContactIdMapping").<Contact, ItemIdMapping>chunk(POPULATE_STEP_CHUNK_SIZE).reader(mongoContactIdMappingItemReader)
.processor(contactIdMappingProcessor()).writer(mongoContactIdMappingItemWriter).build();
}
@Bean
public MongoItemReader<Contact> mongoContactItemReader(MongoTemplate mongoTemplate) {
MongoItemReader<Contact> reader = new MongoItemReader<>();
reader.setTemplate(mongoTemplate);
public Step populateRegistrationIdMappingForEpochPurgeStep(
MongoItemReader<Registration> mongoRegistrationIdMappingForPurgeItemReader,
MongoItemWriter<ItemIdMapping> mongoRegistrationIdMappingItemWriter) {
reader.setSort(new HashMap<String, Sort.Direction>() {{
return this.stepBuilderFactory.get("populateRegistrationIdMappingForPurge").<Registration, ItemIdMapping>chunk(POPULATE_STEP_CHUNK_SIZE).reader(mongoRegistrationIdMappingForPurgeItemReader)
.processor(registrationIdMappingProcessor()).writer(mongoRegistrationIdMappingItemWriter).build();
}
put("_id", Direction.DESC);
@Bean
public Step contactProcessingStep(MongoItemReader<Contact> mongoContactItemReader) {
Step workerStep = contactWorkerStep(stepBuilderFactory, mongoContactItemReader);
}});
return this.stepBuilderFactory.get("readContacts").partitioner("contactWorkerStep", partitioner())
.partitionHandler(partitionHandler(workerStep))
.build();
}
reader.setTargetType(Contact.class);
@Bean
public Step processRegistrationStep(MongoItemReader<Registration> mongoRegistrationItemReader) {
Step workerStep = registrationWorkerStep(stepBuilderFactory, mongoRegistrationItemReader);
reader.setQuery("{}");
return reader;
return this.stepBuilderFactory.get("readRegistrations").partitioner("registrationWorkerStep", partitioner())
.partitionHandler(partitionHandler(workerStep))
.build();
}