Attention une mise à jour du service Gitlab va être effectuée le mardi 30 novembre entre 17h30 et 18h00. Cette mise à jour va générer une interruption du service dont nous ne maîtrisons pas complètement la durée mais qui ne devrait pas excéder quelques minutes. Cette mise à jour intermédiaire en version 14.0.12 nous permettra de rapidement pouvoir mettre à votre disposition une version plus récente.

Commit 8f8486ed authored by Jujube Orange's avatar Jujube Orange
Browse files

refactor(clea-ws): use kafka testcontainer

parent 12c1d531
flyway.connectRetries=60
\ No newline at end of file
......@@ -20,9 +20,9 @@
</description>
<properties>
<hamcrestdate.version>2.0.7</hamcrestdate.version>
<restassured.version>4.3.3</restassured.version>
<springfox-boot-starter.version>3.0.0</springfox-boot-starter.version>
<io.jsonwebtoken.version>0.11.2</io.jsonwebtoken.version>
</properties>
<dependencies>
......@@ -116,6 +116,12 @@
<groupId>org.apache.commons</groupId>
<artifactId>commons-lang3</artifactId>
</dependency>
<dependency>
<groupId>org.exparity</groupId>
<artifactId>hamcrest-date</artifactId>
<version>${hamcrestdate.version}</version>
<scope>test</scope>
</dependency>
</dependencies>
<build>
......
package fr.gouv.clea.ws.service.impl;
import fr.gouv.clea.ws.configuration.CleaKafkaProperties;
import fr.gouv.clea.ws.model.DecodedVisit;
import fr.gouv.clea.ws.model.ReportStat;
import fr.gouv.clea.ws.service.IProducerService;
import fr.gouv.clea.ws.utils.KafkaVisitDeserializer;
import fr.gouv.clea.ws.test.IntegrationTest;
import fr.gouv.clea.ws.test.KafkaManager;
import fr.inria.clea.lsp.EncryptedLocationSpecificPart;
import fr.inria.clea.lsp.utils.TimeUtils;
import org.apache.commons.lang3.RandomUtils;
import org.apache.kafka.clients.consumer.Consumer;
import org.apache.kafka.clients.consumer.ConsumerRecord;
import org.apache.kafka.clients.consumer.ConsumerRecords;
import org.apache.kafka.common.serialization.StringDeserializer;
import org.assertj.core.groups.Tuple;
import org.assertj.core.api.Assertions;
import org.junit.jupiter.api.DisplayName;
import org.junit.jupiter.api.Test;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.boot.test.context.SpringBootTest;
import org.springframework.kafka.core.DefaultKafkaConsumerFactory;
import org.springframework.kafka.support.serializer.JsonDeserializer;
import org.springframework.kafka.test.EmbeddedKafkaBroker;
import org.springframework.kafka.test.context.EmbeddedKafka;
import org.springframework.kafka.test.utils.KafkaTestUtils;
import java.time.Instant;
import java.util.Collections;
import java.util.Base64;
import java.util.List;
import java.util.Map;
import java.util.UUID;
import java.util.stream.Collectors;
import java.util.stream.StreamSupport;
import static org.assertj.core.api.Assertions.assertThat;
import static fr.gouv.clea.ws.test.KafkaRecordAssert.assertThat;
import static org.assertj.core.api.Assertions.tuple;
@SpringBootTest
@EmbeddedKafka(partitions = 1, brokerProperties = { "listeners=PLAINTEXT://localhost:9092", "port=9092" })
@IntegrationTest
class ProducerServiceTest {
@Autowired
private IProducerService producerService;
@Autowired
private EmbeddedKafkaBroker embeddedKafkaBroker;
@Autowired
private CleaKafkaProperties cleaKafkaProperties;
private static DecodedVisit createSerializableDecodedVisit(Instant qrCodeScanTime, boolean isBackward,
UUID locationTemporaryPublicId, byte[] encryptedLocationMessage) {
return new DecodedVisit(
......@@ -64,13 +45,6 @@ class ProducerServiceTest {
@Test
@DisplayName("test that produceVisits can send decoded lsps to kafka and that we can read them back")
void can_send_decrypted_lsps_to_kafka() {
final Map<String, Object> configs = KafkaTestUtils.consumerProps("visitConsumer", "false", embeddedKafkaBroker);
final Consumer<String, DecodedVisit> visitConsumer = new DefaultKafkaConsumerFactory<>(
configs,
new StringDeserializer(),
new KafkaVisitDeserializer()
).createConsumer();
visitConsumer.subscribe(Collections.singleton(cleaKafkaProperties.getQrCodesTopic()));
UUID uuid1 = UUID.randomUUID();
UUID uuid2 = UUID.randomUUID();
......@@ -96,34 +70,36 @@ class ProducerServiceTest {
producerService.produceVisits(decoded);
assertThat(KafkaTestUtils.getRecords(visitConsumer))
final var records = KafkaManager.getRecords();
Assertions.assertThat(records)
.extracting(ConsumerRecord::value)
.extracting(
value -> tuple(
value.getLocationTemporaryPublicId(),
value.getEncryptedLocationSpecificPart().getEncryptedLocationMessage(),
value.getQrCodeScanTime(),
value.isBackward()
value.get("locationTemporaryPublicId").asText(),
value.get("encryptedLocationMessage").asText(),
value.get("qrCodeScanTime").asLong() / 1000,
value.get("isBackward").asBoolean()
)
)
.containsExactly(
Tuple.tuple(uuid1, encryptedLocationMessage1, qrCodeScanTime1, isBackward1),
Tuple.tuple(uuid2, encryptedLocationMessage2, qrCodeScanTime2, isBackward2),
Tuple.tuple(uuid3, encryptedLocationMessage3, qrCodeScanTime3, isBackward3)
tuple(
uuid1.toString(), Base64.getEncoder().encodeToString(encryptedLocationMessage1),
qrCodeScanTime1.getEpochSecond(), isBackward1
),
tuple(
uuid2.toString(), Base64.getEncoder().encodeToString(encryptedLocationMessage2),
qrCodeScanTime2.getEpochSecond(), isBackward2
),
tuple(
uuid3.toString(), Base64.getEncoder().encodeToString(encryptedLocationMessage3),
qrCodeScanTime3.getEpochSecond(), isBackward3
)
);
}
@Test
@DisplayName("test that produceStat can send a stat to kafka and that we can read it back")
void can_send_report_stat_to_kafka() {
final Map<String, Object> configs = KafkaTestUtils.consumerProps("statConsumer", "false", embeddedKafkaBroker);
final Consumer<String, ReportStat> statConsumer = new DefaultKafkaConsumerFactory<>(
configs,
new StringDeserializer(),
new JsonDeserializer<>(ReportStat.class)
).createConsumer();
statConsumer.subscribe(Collections.singleton(cleaKafkaProperties.getStatsTopic()));
long timestamp = TimeUtils.currentNtpTime();
ReportStat reportStat = ReportStat.builder()
......@@ -137,22 +113,16 @@ class ProducerServiceTest {
producerService.produceStat(reportStat);
ConsumerRecords<String, ReportStat> records = KafkaTestUtils.getRecords(statConsumer);
assertThat(records.count()).isEqualTo(1);
List<ReportStat> extracted = StreamSupport
.stream(records.spliterator(), true)
.map(ConsumerRecord::value)
.collect(Collectors.toList());
assertThat(extracted.size()).isEqualTo(1);
var response = extracted.stream().findFirst().get();
assertThat(response.getReported()).isEqualTo(10);
assertThat(response.getRejected()).isEqualTo(2);
assertThat(response.getBackwards()).isEqualTo(5);
assertThat(response.getForwards()).isEqualTo(3);
assertThat(response.getClose()).isEqualTo(4);
assertThat(response.getTimestamp()).isEqualTo(timestamp);
final var record = KafkaManager.getSingleRecord("dev.clea.fct.report-stats");
assertThat(record)
.hasNoKey()
.hasNoHeader("__TypeId__")
.hasJsonValue("reported", 10)
.hasJsonValue("rejected", 2)
.hasJsonValue("backwards", 5)
.hasJsonValue("forwards", 3)
.hasJsonValue("close", 4)
.hasJsonValue("timestamp", timestamp);
}
private Instant newRandomInstant() {
......
......@@ -6,6 +6,7 @@ import ch.qos.logback.core.read.ListAppender;
import fr.gouv.clea.ws.model.DecodedVisit;
import fr.gouv.clea.ws.service.IProducerService;
import fr.gouv.clea.ws.service.IReportService;
import fr.gouv.clea.ws.test.IntegrationTest;
import fr.gouv.clea.ws.vo.ReportRequest;
import fr.gouv.clea.ws.vo.Visit;
import fr.inria.clea.lsp.CleaEciesEncoder;
......@@ -16,7 +17,6 @@ import org.apache.tomcat.util.codec.binary.Base64;
import org.junit.jupiter.api.*;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.boot.test.context.SpringBootTest;
import org.springframework.boot.test.mock.mockito.MockBean;
import java.time.Instant;
......@@ -24,11 +24,13 @@ import java.time.temporal.ChronoUnit;
import java.util.List;
import java.util.UUID;
import static java.time.temporal.ChronoUnit.SECONDS;
import static org.assertj.core.api.Assertions.assertThat;
import static org.assertj.core.api.Assertions.tuple;
import static org.mockito.ArgumentMatchers.anyList;
import static org.mockito.Mockito.doNothing;
@SpringBootTest
@IntegrationTest
class ReportServiceTest {
@MockBean
......@@ -41,7 +43,7 @@ class ReportServiceTest {
@BeforeEach
void init() {
now = Instant.now();
now = Instant.now().truncatedTo(SECONDS);
assertThat(producerService).isNotNull();
assertThat(reportService).isNotNull();
doNothing().when(producerService).produceVisits(anyList());
......@@ -115,7 +117,7 @@ class ReportServiceTest {
UUID uuid2 = UUID.randomUUID();
List<Visit> visits = List.of(
newVisit(uuid1, TimeUtils.ntpTimestampFromInstant(now)), // pass
newVisit(uuid2, TimeUtils.ntpTimestampFromInstant(now.plus(2, ChronoUnit.SECONDS))) /* don't pass */
newVisit(uuid2, TimeUtils.ntpTimestampFromInstant(now.plus(2, SECONDS))) /* don't pass */
);
List<DecodedVisit> processed = reportService.report(new ReportRequest(visits, 0L));
......@@ -136,11 +138,14 @@ class ReportServiceTest {
UUID uuidC = UUID.fromString("bdbf9725-c1ad-42e3-b725-e475272b7f54");
UUID uuidC2 = UUID.fromString("bdbf9725-c1ad-42e3-b725-e475272b7f54");
final var nowMinus4h = now.minus(4, ChronoUnit.HOURS).truncatedTo(SECONDS);
final var nowPlus3h = now.minus(3, ChronoUnit.HOURS).truncatedTo(SECONDS);
List<Visit> visits = List.of(
newVisit(uuidA, TimeUtils.ntpTimestampFromInstant(now.minus(4, ChronoUnit.HOURS))), // pass
newVisit(uuidA, TimeUtils.ntpTimestampFromInstant(nowMinus4h)), // pass
newVisit(uuidA2, TimeUtils.ntpTimestampFromInstant(now)), // pass
newVisit(uuidA2, TimeUtils.ntpTimestampFromInstant(now.plus(15, ChronoUnit.MINUTES))), // don't pass
newVisit(uuidB, TimeUtils.ntpTimestampFromInstant(now.minus(3, ChronoUnit.HOURS))), // pass
newVisit(uuidB, TimeUtils.ntpTimestampFromInstant(nowPlus3h)), // pass
newVisit(uuidB2, TimeUtils.ntpTimestampFromInstant(now)), // don't pass
newVisit(uuidB, TimeUtils.ntpTimestampFromInstant(now.minus(1, ChronoUnit.HOURS))), // don't pass
newVisit(uuidC, TimeUtils.ntpTimestampFromInstant(now)), // pass
......@@ -151,8 +156,13 @@ class ReportServiceTest {
List<DecodedVisit> processed = reportService.report(new ReportRequest(visits, 0L));
assertThat(processed)
.extracting(DecodedVisit::getLocationTemporaryPublicId)
.containsExactly(uuidA, uuidA, uuidB, uuidC);
.extracting(v -> tuple(v.getLocationTemporaryPublicId(), v.getQrCodeScanTime()))
.containsExactly(
tuple(uuidA, nowMinus4h),
tuple(uuidA, now),
tuple(uuidB, nowPlus3h),
tuple(uuidC, now)
);
}
@Test
......@@ -240,110 +250,4 @@ class ReportServiceTest {
String qrCode = Base64.encodeBase64URLSafeString(encodedQr);
return new Visit(qrCode, qrCodeScanTime);
}
@Nested
@DisplayNameGeneration(DisplayNameGenerator.ReplaceUnderscores.class)
class LoggingTest {
private ListAppender<ILoggingEvent> loggingEventListAppender;
@BeforeEach
void setUp() {
now = Instant.now();
this.loggingEventListAppender = new ListAppender<>();
this.loggingEventListAppender.start();
((Logger) LoggerFactory.getLogger(ReportService.class)).addAppender(loggingEventListAppender);
}
@Test
void test_that_duplicate_visits_increment_rejected_visits_count() {
UUID uuidA = UUID.randomUUID();
UUID uuidB = UUID.randomUUID();
UUID uuidC = UUID.randomUUID();
List<Visit> visits = List.of(
newVisit(uuidA, TimeUtils.ntpTimestampFromInstant(now.minus(4, ChronoUnit.HOURS))), // pass
newVisit(uuidA, TimeUtils.ntpTimestampFromInstant(now)), // pass
newVisit(uuidB, TimeUtils.ntpTimestampFromInstant(now.minus(3, ChronoUnit.HOURS))), // pass
newVisit(uuidB, TimeUtils.ntpTimestampFromInstant(now)), // don't pass
newVisit(uuidC, TimeUtils.ntpTimestampFromInstant(now)), // pass
newVisit(uuidC, TimeUtils.ntpTimestampFromInstant(now)) /* don't pass */
);
reportService.report(new ReportRequest(visits, 0L));
List<ILoggingEvent> logsList = loggingEventListAppender.list;
assertThat(logsList).extracting("formattedMessage")
.contains(String.format("BATCH_REPORT %s#%s#%s#%s#%s", visits.size(), 2, 0, 4, 0));
}
@Test
void test_that_backward_visit_increments_backward_visits_count() {
long pivotDate = TimeUtils.ntpTimestampFromInstant(now);
long qrScan = TimeUtils.ntpTimestampFromInstant(now.minus(1, ChronoUnit.DAYS));
UUID uuid = UUID.randomUUID();
List<Visit> visits = List.of(newVisit(uuid, qrScan));
reportService.report(new ReportRequest(visits, pivotDate));
List<ILoggingEvent> logsList = loggingEventListAppender.list;
assertThat(logsList).extracting("formattedMessage")
.contains(String.format("BATCH_REPORT %s#%s#%s#%s#%s", visits.size(), 0, 1, 0, 0));
}
@Test
void test_that_forward_visit_increments_forward_visits_count() {
long pivotDate = TimeUtils.ntpTimestampFromInstant(now.minus(1, ChronoUnit.DAYS));
long qrScan = TimeUtils.ntpTimestampFromInstant(now);
UUID uuid1 = UUID.randomUUID();
UUID uuid2 = UUID.randomUUID();
List<Visit> visits = List.of(
newVisit(uuid1, qrScan),
newVisit(uuid2, pivotDate)
);
reportService.report(new ReportRequest(visits, pivotDate));
List<ILoggingEvent> logsList = loggingEventListAppender.list;
assertThat(logsList).extracting("formattedMessage")
.contains(String.format("BATCH_REPORT %s#%s#%s#%s#%s", visits.size(), 0, 0, 2, 0));
}
@Test
void test_that_outdated_visits_increments_rejected_visits_count() {
UUID uuid1 = UUID.randomUUID();
UUID uuid2 = UUID.randomUUID();
UUID uuid3 = UUID.randomUUID();
UUID uuid4 = UUID.randomUUID();
List<Visit> visits = List.of(
newVisit(uuid1, TimeUtils.ntpTimestampFromInstant(now.minus(15, ChronoUnit.DAYS))), // don't pass
newVisit(uuid2, TimeUtils.ntpTimestampFromInstant(now.minus(13, ChronoUnit.DAYS))), // pass
newVisit(uuid3, TimeUtils.ntpTimestampFromInstant(now.minus(2, ChronoUnit.DAYS))), // pass
newVisit(uuid4, TimeUtils.ntpTimestampFromInstant(now)) /* pass */
);
ReportRequest reportRequestVo = new ReportRequest(visits, 0L);
reportService.report(reportRequestVo);
List<ILoggingEvent> logsList = loggingEventListAppender.list;
assertThat(logsList).extracting("formattedMessage")
.contains(String.format("BATCH_REPORT %s#%s#%s#%s#%s", visits.size(), 1, 0, 3, 0));
}
@Test
void test_that_future_visits_increments_rejected_visits_count() {
UUID uuid1 = UUID.randomUUID();
UUID uuid2 = UUID.randomUUID();
List<Visit> visits = List.of(
newVisit(uuid1, TimeUtils.ntpTimestampFromInstant(now)), // pass
newVisit(uuid2, TimeUtils.ntpTimestampFromInstant(now.plus(2, ChronoUnit.SECONDS))) // don't pass
);
reportService.report(new ReportRequest(visits, 0L));
List<ILoggingEvent> logsList = loggingEventListAppender.list;
assertThat(logsList).extracting("formattedMessage")
.contains(String.format("BATCH_REPORT %s#%s#%s#%s#%s", visits.size(), 1, 0, 1, 0));
}
}
}
......@@ -60,11 +60,12 @@ public class KafkaManager implements TestExecutionListener {
@Override
public void afterTestMethod(TestContext testContext) {
consumer.commitSync();
KafkaTestUtils.getRecords(consumer, 1000);
consumer.close();
}
public static ConsumerRecords<String, JsonNode> getRecords() {
return KafkaTestUtils.getRecords(consumer);
return KafkaTestUtils.getRecords(consumer, 100);
}
public static ConsumerRecord<String, JsonNode> getSingleRecord(String topic) {
......
package fr.gouv.clea.ws.test;
import fr.inria.clea.lsp.Location;
import fr.inria.clea.lsp.LocationContact;
import fr.inria.clea.lsp.LocationSpecificPart;
import fr.inria.clea.lsp.LocationSpecificPartDecoder;
import org.bouncycastle.util.encoders.Hex;
import java.net.URL;
import java.time.Instant;
import java.util.Base64;
import java.util.Random;
import java.util.UUID;
import static java.time.temporal.ChronoUnit.DAYS;
import static java.time.temporal.ChronoUnit.HOURS;
public class QrCode {
// server authority public key
private static final String PK_SA = "02c3a58bf668fa3fe2fc206152abd6d8d55102adfee68c8b227676d1fe763f5a06";
// manual contact tracing authority public key
private static final String PK_MCTA = "02c3a58bf668fa3fe2fc206152abd6d8d55102adfee68c8b227676d1fe763f5a06";
public static Location LOCATION_1;
public static URL LOCATION_1_URL;
public static String LOCATION_1_LOCATION_SPECIFIC_PART_DECODED_BASE64;
public static UUID LOCATION_1_LOCATION_TEMPORARY_SPECIFIC_ID;
public static Location LOCATION_2;
public static URL LOCATION_2_URL;
static {
final var instant = Instant.now()
.minus(365, DAYS)
.truncatedTo(HOURS);
LOCATION_1 = createRandomLocation(instant);
LOCATION_2 = createRandomLocation(instant);
try {
LOCATION_1_URL = new URL(LOCATION_1.newDeepLink());
LOCATION_1_LOCATION_SPECIFIC_PART_DECODED_BASE64 = Base64.getEncoder().encodeToString(
new LocationSpecificPartDecoder()
.decodeHeader(Base64.getUrlDecoder().decode(LOCATION_1_URL.getRef()))
.getEncryptedLocationMessage()
);
LOCATION_1_LOCATION_TEMPORARY_SPECIFIC_ID = LOCATION_1.getLocationSpecificPart()
.getLocationTemporaryPublicId();
LOCATION_2_URL = new URL(LOCATION_2.newDeepLink());
} catch (Exception e) {
throw new RuntimeException(e);
}
}
private static Location createRandomLocation(Instant instant) {
return Location.builder()
.manualContactTracingAuthorityPublicKey(PK_MCTA)
.serverAuthorityPublicKey(PK_SA)
.permanentLocationSecretKey(generatePermanentLocationSecretKey())
.locationSpecificPart(createRandomLocationSpecificPart())
.contact(
LocationContact.builder()
.locationPhone("01000000")
.locationRegion(0)
.locationPin("123456")
.periodStartTime(instant)
.build()
)
.build();
}
private static LocationSpecificPart createRandomLocationSpecificPart() {
return LocationSpecificPart.builder()
.staff(false)
.periodDuration(255 /* unlimited */)
.qrCodeRenewalIntervalExponentCompact(0x1F /* no renewal */)
.venueType(1)
.venueCategory1(1)
.venueCategory2(1)
.build();
}
private static String generatePermanentLocationSecretKey() {
final var permanentLocationSecretKey = new byte[LocationSpecificPart.LOCATION_TEMPORARY_SECRET_KEY_SIZE];
new Random().nextBytes(permanentLocationSecretKey);
return Hex.toHexString(permanentLocationSecretKey);
}
}
package fr.gouv.clea.ws.test;
import org.exparity.hamcrest.date.ZonedDateTimeMatchers;
import org.hamcrest.Description;
import org.hamcrest.Matcher;
import org.hamcrest.Matchers;
import org.hamcrest.TypeSafeMatcher;
import java.time.ZonedDateTime;
public class TemporalMatchers {
/**
* Hamcrest matcher to verify a string representation of a datetime is between
* now and 10 seconds ago.
*/
public static Matcher<String> isStringDateBetweenNowAndTenSecondsAgo() {
final var dateTimeBetweenNowAndOneSecondAgo = isBetweenNowAndTenSecondsAgo();
return new TypeSafeMatcher<>() {
@Override
protected boolean matchesSafely(String value) {
final var actualDate = ZonedDateTime.parse(value);
return dateTimeBetweenNowAndOneSecondAgo.matches(actualDate);
}
@Override
public void describeTo(Description description) {
dateTimeBetweenNowAndOneSecondAgo.describeTo(description);
}
};
}
/**
* Hamcrest matcher to verify a {@link ZonedDateTime} is between now and 10
* seconds ago.
*/
private static Matcher<ZonedDateTime> isBetweenNowAndTenSecondsAgo() {
return Matchers.allOf(
ZonedDateTimeMatchers.sameOrAfter(ZonedDateTime.now().minusSeconds(10)),
ZonedDateTimeMatchers.sameOrBefore(ZonedDateTime.now())
);
}
}
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment