Skip to content

GitLab

Commit 6ca6a26d authored by Jujube Orange's avatar Jujube Orange
Browse files

refactor(ws-rest): report & visit validation

parent c49fcea5
Pipeline #283483 failed with stages
in 8 minutes and 4 seconds
Hide whitespace changes
Inline Side-by-side
.gitlab-ci.yml
View file @ 6ca6a26d
......@@ -38,7 +38,7 @@ integration-tests:
- docker-compose up -d
- docker network connect clea-server_default $currentContainer
- echo "Waiting for clea-ws-rest component to be up..."
- until curl -s http://clea-ws-rest:8080/actuator/health | grep -q UP; do sleep 0.5; done;
- until curl -s http://clea-ws-rest:8081/actuator/health | grep -q UP; do sleep 0.5; done;
- java -jar -Dspring.profiles.active=docker clea-integration-tests/target/clea-integration-tests-*.jar
- docker network disconnect clea-server_default $currentContainer
- docker-compose down
......
clea-dbmigration/docker-compose/docker-compose.yml
View file @ 6ca6a26d
......@@ -2,7 +2,7 @@ version: "3"
services:
flyway:
image: flyway/flyway:7.8
#command: migrate
#command: migrate -connectRetries=60
#command: migrate -baselineOnMigrate=true -baselineVersion=4
entrypoint: ["tail", "-f", "/dev/null"]
environment:
......@@ -10,8 +10,7 @@ services:
FLYWAY_USER: postgres
FLYWAY_PASSWORD: password
volumes:
- ../conf/:/flyway/conf
- ../sql/:/flyway/sql
- ../src/main/resources/db/migrations:/flyway/sql
depends_on:
- postgres
......
clea-integration-tests/src/main/java/fr/gouv/clea/integrationtests/service/CleaBatchService.java
View file @ 6ca6a26d
......@@ -3,6 +3,7 @@ package fr.gouv.clea.integrationtests.service;
import fr.gouv.clea.integrationtests.config.ApplicationProperties;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.io.IOUtils;
import org.springframework.stereotype.Service;
import java.io.BufferedReader;
......@@ -30,10 +31,12 @@ public class CleaBatchService {
final var batchTriggerCommand = applicationProperties.getBatch().getCommand().split(" ");
final var builder = new ProcessBuilder(batchTriggerCommand);
builder.directory(Path.of(".").toFile());
var process = builder.start();
var streamGobbler = new StreamGobbler(process.getInputStream(), log::debug);
Executors.newSingleThreadExecutor().submit(streamGobbler);
final var process = builder.start();
final var background = Executors.newFixedThreadPool(2);
background.submit(new StreamGobbler(process.getInputStream(), log::info));
background.submit(new StreamGobbler(process.getErrorStream(), log::info));
boolean hasExited = process.waitFor(BATCH_EXECUTION_TIMEOUT_IN_SECONDS, TimeUnit.SECONDS);
background.shutdownNow();
if (!hasExited) {
throw new RuntimeException("Cluster detection trigger timeout");
}
......@@ -55,7 +58,8 @@ public class CleaBatchService {
@Override
public void run() {
new BufferedReader(new InputStreamReader(inputStream)).lines()
new BufferedReader(new InputStreamReader(inputStream))
.lines()
.forEach(consumer);
}
}
......
clea-venue-consumer/src/main/resources/application-dev.yml
View file @ 6ca6a26d
......@@ -24,9 +24,9 @@ clea:
risk:
enabled: "false"
kafka:
qrCodesTopic: cleaQrCodes
reportStatsTopic: cleaStats
errorLocationStatsTopic: cleaErrorStats
qrCodesTopic: dev.clea.fct.visit-scans
reportStatsTopic: dev.clea.fct.report-stats
errorLocationStatsTopic: dev.clea.fct.clea-error-visit
management:
endpoints:
......
clea-ws-rest/src/main/java/fr/gouv/clea/ws/configuration/SecurityConfiguration.java
View file @ 6ca6a26d
package fr.gouv.clea.ws.configuration;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import org.springframework.boot.actuate.autoconfigure.security.servlet.EndpointRequest;
import org.springframework.context.annotation.Bean;
import org.springframework.security.config.annotation.web.builders.HttpSecurity;
......@@ -19,6 +20,7 @@ import java.util.Base64;
import static org.springframework.security.config.http.SessionCreationPolicy.STATELESS;
import static org.springframework.security.oauth2.jose.jws.SignatureAlgorithm.RS256;
@Slf4j
@EnableWebSecurity
@RequiredArgsConstructor
public class SecurityConfiguration extends WebSecurityConfigurerAdapter {
......@@ -28,14 +30,21 @@ public class SecurityConfiguration extends WebSecurityConfigurerAdapter {
@Override
protected void configure(final HttpSecurity http) throws Exception {
http
.sessionManagement().sessionCreationPolicy(STATELESS);
http.oauth2ResourceServer()
.jwt();
http.authorizeRequests()
.requestMatchers(EndpointRequest.toAnyEndpoint()).permitAll()
.anyRequest().authenticated();
.sessionManagement().sessionCreationPolicy(STATELESS)
.and()
.csrf().disable();
if (cleaWsProperties.isAuthorizationCheckActive()) {
http.oauth2ResourceServer()
.jwt();
http.authorizeRequests()
.requestMatchers(EndpointRequest.toAnyEndpoint()).permitAll()
.anyRequest().authenticated();
} else {
log.warn("Authentication is disabled");
http.authorizeRequests()
.anyRequest().permitAll();
}
}
@Bean
......
clea-ws-rest/src/main/java/fr/gouv/clea/ws/controller/CleaController.java
View file @ 6ca6a26d
package fr.gouv.clea.ws.controller;
import fr.gouv.clea.ws.api.CleaApi;
import fr.gouv.clea.ws.api.model.ReportRequest;
import fr.gouv.clea.ws.api.model.ReportResponse;
import fr.gouv.clea.ws.api.model.ValidationError;
import fr.gouv.clea.ws.exception.CleaBadRequestException;
import fr.gouv.clea.ws.model.DecodedVisit;
import fr.gouv.clea.ws.service.IReportService;
import fr.gouv.clea.ws.utils.BadArgumentsLoggerService;
import fr.gouv.clea.ws.vo.ReportRequest;
import fr.gouv.clea.ws.vo.Visit;
import fr.gouv.clea.ws.service.ReportService;
import fr.gouv.clea.ws.service.model.Visit;
import fr.inria.clea.lsp.utils.TimeUtils;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import org.springframework.http.ResponseEntity;
import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.RestController;
import org.springframework.web.context.request.WebRequest;
import javax.validation.ConstraintViolation;
import javax.validation.Validator;
import java.util.Collections;
import java.util.HashSet;
import java.util.List;
import java.util.Set;
import static java.util.stream.Collectors.toList;
import static org.springframework.util.CollectionUtils.isEmpty;
@RestController
@RequestMapping(path = "/api/clea/v1")
......@@ -31,67 +23,62 @@ import static java.util.stream.Collectors.toList;
@Slf4j
public class CleaController implements CleaApi {
public static final String MALFORMED_VISIT_LOG_MESSAGE = "Filtered out %d malformed visits of %d while Exposure Status Request";
private final ReportService reportService;
@Override
public ResponseEntity<ReportResponse> reportUsingPOST(ReportRequest reportRequest) {
nonNullPivotDateOrThrowBadRequest(reportRequest);
nonEmptyVisitsOrThrowBadRequest(reportRequest);
private final IReportService reportService;
final var pivotDate = TimeUtils.instantFromTimestamp(reportRequest.getPivotDate());
final var visits = reportRequest.getVisits()
.stream()
.map(this::toVisitNullSafe)
.collect(toList());
private final BadArgumentsLoggerService badArgumentsLoggerService;
final var acceptedVisits = reportService.report(pivotDate, visits);
final var message = String.format("%d/%d accepted visits", acceptedVisits, reportRequest.getVisits().size());
log.info(message);
private final WebRequest webRequest;
if (acceptedVisits > 0) {
return ResponseEntity.ok(
ReportResponse.builder()
.success(true)
.message(message)
.build()
);
} else {
throw new CleaBadRequestException(message);
}
}
private final Validator validator;
private Visit toVisitNullSafe(fr.gouv.clea.ws.api.model.Visit visit) {
return visit == null ? null : new Visit(visit.getQrCode(), visit.getQrCodeScanTime());
}
@Override
public ResponseEntity<ReportResponse> reportUsingPOST(fr.gouv.clea.ws.api.model.ReportRequest reportRequest) {
final var visits = reportRequest.getVisits() == null ? Collections.<Visit>emptyList()
: reportRequest.getVisits().stream()
.map(visit -> new Visit(visit.getQrCode(), visit.getQrCodeScanTime()))
.collect(toList());
final var reportRequestVo = new ReportRequest(
visits,
reportRequest.getPivotDate()
);
ReportRequest filtered = this.filterReports(reportRequestVo, webRequest);
List<DecodedVisit> reported = List.of();
if (!filtered.getVisits().isEmpty()) {
reported = reportService.report(filtered);
private void nonNullPivotDateOrThrowBadRequest(ReportRequest reportRequest) {
if (reportRequest.getPivotDate() == null) {
throw new CleaBadRequestException(
ValidationError.builder()
.rejectedValue(null)
._object("ReportRequest")
.message("must not be null")
.field("pivotDate")
.build()
);
}
String message = String.format(
"%s reports processed, %s rejected", reported.size(),
reportRequestVo.getVisits().size() - reported.size()
);
log.info(message);
return ResponseEntity.ok(new ReportResponse(message, true));
}
private ReportRequest filterReports(ReportRequest report, WebRequest webRequest) {
Set<ConstraintViolation<ReportRequest>> reportRequestViolations = validator.validate(report);
if (!reportRequestViolations.isEmpty()) {
throw new CleaBadRequestException(reportRequestViolations, Set.of());
} else {
Set<ConstraintViolation<Visit>> visitViolations = new HashSet<>();
List<Visit> validVisits = report.getVisits().stream()
.filter(
visit -> {
visitViolations.addAll(validator.validate(visit));
if (!visitViolations.isEmpty()) {
this.badArgumentsLoggerService
.logValidationErrorMessage(visitViolations, webRequest);
return false;
} else {
return true;
}
}
).collect(toList());
if (validVisits.isEmpty()) {
throw new CleaBadRequestException(Set.of(), visitViolations);
}
int nbVisits = report.getVisits().size();
int nbFilteredVisits = nbVisits - validVisits.size();
if (nbFilteredVisits > 0) {
log.warn(String.format(MALFORMED_VISIT_LOG_MESSAGE, nbFilteredVisits, nbVisits));
}
return new ReportRequest(validVisits, report.getPivotDateAsNtpTimestamp());
private void nonEmptyVisitsOrThrowBadRequest(ReportRequest reportRequest) {
if (isEmpty(reportRequest.getVisits())) {
throw new CleaBadRequestException(
ValidationError.builder()
.rejectedValue(reportRequest.getVisits())
._object("ReportRequest")
.message("must not be empty")
.field("visits")
.build()
);
}
}
}
clea-ws-rest/src/main/java/fr/gouv/clea/ws/exception/AbstractCleaException.java deleted 100644 → 0
View file @ c49fcea5
package fr.gouv.clea.ws.exception;
import lombok.Data;
import lombok.EqualsAndHashCode;
import lombok.NoArgsConstructor;
import java.time.Instant;
@Data
@NoArgsConstructor
@EqualsAndHashCode(callSuper = false)
public abstract class AbstractCleaException extends RuntimeException {
protected String code;
protected Instant timestamp;
protected AbstractCleaException(String message, String code) {
super(message);
this.code = code;
this.timestamp = Instant.now();
}
}
clea-ws-rest/src/main/java/fr/gouv/clea/ws/exception/CleaBadRequestException.java
View file @ 6ca6a26d
package fr.gouv.clea.ws.exception;
import fr.gouv.clea.ws.vo.ReportRequest;
import fr.gouv.clea.ws.vo.Visit;
import lombok.Data;
import lombok.EqualsAndHashCode;
import org.springframework.http.HttpStatus;
import org.springframework.web.bind.annotation.ResponseStatus;
import fr.gouv.clea.ws.api.model.ValidationError;
import lombok.Getter;
import javax.validation.ConstraintViolation;
import java.util.Collections;
import java.util.List;
import java.util.Set;
public class CleaBadRequestException extends RuntimeException {
@Data
@EqualsAndHashCode(callSuper = true)
@ResponseStatus(HttpStatus.BAD_REQUEST)
public class CleaBadRequestException extends AbstractCleaException {
@Getter
private final List<ValidationError> validationErrors;
private static final String EX_CODE = "clea-003";
private static final String MESSAGE = "Invalid request";
private Set<ConstraintViolation<ReportRequest>> reportRequestViolations;
private Set<ConstraintViolation<Visit>> visitViolations;
public CleaBadRequestException(String message) {
super(message);
this.validationErrors = Collections.emptyList();
}
public CleaBadRequestException(
Set<ConstraintViolation<ReportRequest>> reportRequestViolations,
Set<ConstraintViolation<Visit>> visitViolations) {
super(MESSAGE, EX_CODE);
this.reportRequestViolations = reportRequestViolations;
this.visitViolations = visitViolations;
public CleaBadRequestException(ValidationError error) {
super("Invalid request");
validationErrors = List.of(error);
}
}
clea-ws-rest/src/main/java/fr/gouv/clea/ws/exception/CleaForbiddenException.java deleted 100644 → 0
View file @ c49fcea5
package fr.gouv.clea.ws.exception;
import lombok.Data;
import lombok.EqualsAndHashCode;
import org.springframework.http.HttpStatus;
import org.springframework.web.bind.annotation.ResponseStatus;
@Data
@EqualsAndHashCode(callSuper = true)
@ResponseStatus(HttpStatus.FORBIDDEN)
public class CleaForbiddenException extends AbstractCleaException {
private static final String EX_CODE = "clea-001";
private static final String MESSAGE = "Could not be authenticated (Authorisation header/token invalid)";
public CleaForbiddenException() {
super(MESSAGE, EX_CODE);
}
}
clea-ws-rest/src/main/java/fr/gouv/clea/ws/exception/CleaKafkaException.java deleted 100644 → 0
View file @ c49fcea5
package fr.gouv.clea.ws.exception;
import lombok.Data;
import lombok.EqualsAndHashCode;
import org.springframework.http.HttpStatus;
import org.springframework.web.bind.annotation.ResponseStatus;
@Data
@EqualsAndHashCode(callSuper = true)
@ResponseStatus(HttpStatus.SERVICE_UNAVAILABLE)
public class CleaKafkaException extends AbstractCleaException {
private static final String EX_CODE = "clea-004";
private static final String MESSAGE = "Could not persist request (Queue down)";
public CleaKafkaException() {
super(MESSAGE, EX_CODE);
}
}
clea-ws-rest/src/main/java/fr/gouv/clea/ws/exception/CleaRestExceptionHandler.java
View file @ 6ca6a26d
......@@ -5,7 +5,6 @@ import fr.gouv.clea.ws.api.model.ValidationError;
import lombok.extern.slf4j.Slf4j;
import org.springframework.http.HttpHeaders;
import org.springframework.http.HttpStatus;
import org.springframework.http.MediaType;
import org.springframework.http.ResponseEntity;
import org.springframework.http.converter.HttpMessageNotReadableException;
import org.springframework.web.bind.MethodArgumentNotValidException;
......@@ -15,14 +14,14 @@ import org.springframework.web.bind.annotation.ResponseStatus;
import org.springframework.web.context.request.WebRequest;
import org.springframework.web.servlet.mvc.method.annotation.ResponseEntityExceptionHandler;
import java.time.Instant;
import java.time.OffsetDateTime;
import java.util.List;
import java.util.Set;
import java.util.stream.Collectors;
import java.util.stream.Stream;
import static java.time.ZoneOffset.UTC;
import static java.util.stream.Collectors.toList;
import static org.springframework.http.HttpStatus.BAD_REQUEST;
@ControllerAdvice
@Slf4j
......@@ -33,9 +32,16 @@ public class CleaRestExceptionHandler extends ResponseEntityExceptionHandler {
@ExceptionHandler(CleaBadRequestException.class)
public ResponseEntity<ErrorResponse> handleCleaBadRequestException(CleaBadRequestException ex,
WebRequest webRequest) {
final HttpStatus status = getHttpStatus(ex);
log.error(String.format(ERROR_MESSAGE_TEMPLATE, ex.getLocalizedMessage(), webRequest.getDescription(false)));
return this.jsonResponseEntity(this.cleaBadRequestExceptionToApiError(ex, status));
return ResponseEntity.badRequest()
.body(
new ErrorResponse(
BAD_REQUEST.value(),
Instant.now().atOffset(UTC),
ex.getLocalizedMessage(),
ex.getValidationErrors()
)
);
}
@Override
......@@ -89,7 +95,9 @@ public class CleaRestExceptionHandler extends ResponseEntityExceptionHandler {
log.error(
String.format(ERROR_MESSAGE_TEMPLATE, ex.getLocalizedMessage(), webRequest.getDescription(false)), ex
);
return this.jsonResponseEntity(this.exceptionToApiError(ex, status));
return ResponseEntity
.status(status)
.body(this.exceptionToApiError(ex, status));
}
private HttpStatus getHttpStatus(Exception ex) {
......@@ -97,56 +105,12 @@ public class CleaRestExceptionHandler extends ResponseEntityExceptionHandler {
return responseStatus == null ? HttpStatus.INTERNAL_SERVER_ERROR : responseStatus.value();
}
private ResponseEntity<ErrorResponse> jsonResponseEntity(ErrorResponse apiError) {
HttpHeaders headers = new HttpHeaders();
headers.setContentType(MediaType.APPLICATION_JSON);
return new ResponseEntity<>(apiError, headers, apiError.getHttpStatus());
}
private ErrorResponse exceptionToApiError(Exception ex, HttpStatus status) {
return new ErrorResponse(
status.value(),
ex instanceof AbstractCleaException ? ((AbstractCleaException) ex).getTimestamp().atOffset(UTC)
: OffsetDateTime.now(),
OffsetDateTime.now(),
ex.getLocalizedMessage(),
List.of()
);
}
private ErrorResponse cleaBadRequestExceptionToApiError(CleaBadRequestException ex, HttpStatus status) {
final String splitRegex = "\\.";
Set<ValidationError> superErrors = ex.getReportRequestViolations().stream().map(
it -> {
String[] objectSplits = it.getRootBeanClass().getName().split(splitRegex);
String[] fieldSplits = it.getPropertyPath().toString().split(splitRegex);
return new ValidationError(
objectSplits[objectSplits.length - 1],
fieldSplits[fieldSplits.length - 1],
it.getInvalidValue(),
it.getMessage()
);
}
).collect(Collectors.toSet());
Set<ValidationError> subErrors = ex.getVisitViolations().stream().map(
it -> {
String[] objectSplits = it.getRootBeanClass().getName().split(splitRegex);
String[] fieldSplits = it.getPropertyPath().toString().split(splitRegex);
return new ValidationError(
objectSplits[objectSplits.length - 1],
fieldSplits[fieldSplits.length - 1],
it.getInvalidValue(),
it.getMessage()
);
}
).collect(Collectors.toSet());
return new ErrorResponse(
status.value(),
ex.getTimestamp().atOffset(UTC),
ex.getLocalizedMessage(),
Stream.concat(
superErrors.stream(),
subErrors.stream()
).collect(toList())
);
}
}
clea-ws-rest/src/main/java/fr/gouv/clea/ws/exception/CleaUnauthorizedException.java deleted 100644 → 0
View file @ c49fcea5
package fr.gouv.clea.ws.exception;
import lombok.Data;
import lombok.EqualsAndHashCode;
import org.springframework.http.HttpStatus;
import org.springframework.web.bind.annotation.ResponseStatus;
@Data
@EqualsAndHashCode(callSuper = false)
@ResponseStatus(HttpStatus.UNAUTHORIZED)
public class CleaUnauthorizedException extends AbstractCleaException {
private static final String EX_CODE = "clea-002";
private static final String MESSAGE = "Could not be authorized (Missing authorisation header/token)";
public CleaUnauthorizedException() {
super(MESSAGE, EX_CODE);
}
}
clea-ws-rest/src/main/java/fr/gouv/clea/ws/model/DecodedVisit.java
View file @ 6ca6a26d
package fr.gouv.clea.ws.model;
import fr.inria.clea.lsp.EncryptedLocationSpecificPart;
import lombok.AllArgsConstructor;
import lombok.Getter;
import lombok.ToString;
import lombok.*;
import java.time.Instant;
import java.util.UUID;
@AllArgsConstructor
@Getter
@ToString
@Builder
@Value
public class DecodedVisit {
private final Instant qrCodeScanTime; // t_qrScan
Instant qrCodeScanTime; // t_qrScan
private final EncryptedLocationSpecificPart encryptedLocationSpecificPart;
EncryptedLocationSpecificPart encryptedLocationSpecificPart;
private final boolean isBackward;
boolean isBackward;
public UUID getLocationTemporaryPublicId() {
return this.encryptedLocationSpecificPart.getLocationTemporaryPublicId();
......
clea-ws-rest/src/main/java/fr/gouv/clea/ws/service/IProducerService.java deleted 100644 → 0
View file @ c49fcea5
package fr.gouv.clea.ws.service;
import fr.gouv.clea.ws.model.DecodedVisit;
import fr.gouv.clea.ws.model.ReportStat;
import java.util.List;
public interface IProducerService {
void produceVisits(List<DecodedVisit> decodedVisits);
void produceStat(ReportStat reportStat);
}