Commit c334f9cb authored by Jujube Orange's avatar Jujube Orange
Browse files

Merge branch 'develop'

parents 493211d3 189039ff
Pipeline #257319 canceled with stages
HELP.md
.git-versioned-pom.xml
target/
!.mvn/wrapper/maven-wrapper.jar
!**/src/main/**/target/
......
variables:
MAVEN_OPTS: "-Dmaven.repo.local=.m2/repository"
VERSIONING_DISABLE: "false"
VERSIONING_GIT_TAG: $CI_COMMIT_TAG
VERSIONING_GIT_BRANCH: $CI_COMMIT_BRANCH
default:
cache:
......
<extensions xmlns="http://maven.apache.org/EXTENSIONS/1.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/EXTENSIONS/1.0.0 http://maven.apache.org/xsd/core-extensions-1.0.0.xsd">
<extension>
<groupId>me.qoomon</groupId>
<artifactId>maven-git-versioning-extension</artifactId>
<version>6.4.4</version>
</extension>
</extensions>
<configuration xmlns="https://github.com/qoomon/maven-git-versioning-extension" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="https://github.com/qoomon/maven-git-versioning-extension https://qoomon.github.io/maven-git-versioning-extension/configuration-6.4.0.xsd">
<disable>true</disable>
<preferTags>true</preferTags>
<branch>
<pattern>(.+)</pattern>
<versionFormat>${version}</versionFormat>
</branch>
<tag>
<pattern>(.+)</pattern>
<versionFormat>${tag.slug}</versionFormat>
</tag>
</configuration>
......@@ -3,7 +3,7 @@
<parent>
<artifactId>clea-server</artifactId>
<groupId>fr.gouv.clea</groupId>
<version>1.1.2</version>
<version>0-SNAPSHOT</version>
</parent>
<modelVersion>4.0.0</modelVersion>
......
......@@ -11,7 +11,7 @@ CREATE TABLE exposed_visits (
created_at TIMESTAMP NOT NULL DEFAULT now(),
updated_at TIMESTAMP NOT NULL DEFAULT now(),
PRIMARY KEY (id)
);
CREATE UNIQUE INDEX IF NOT EXISTS exposed_visits_ltidperiodslots ON exposed_visits (ltid, period_start, timeslot);
......@@ -49,3 +49,15 @@ CREATE TABLE stat_location
);
CREATE INDEX IF NOT EXISTS statloc_venue ON stat_location(venue_type, venue_category1, venue_category2);
-- Needs: Clea-Statistiques
CREATE TABLE stat_reports
(
id BIGINT NOT NULL,
backwards INT NOT NULL,
close INT NOT NULL,
forwards INT NOT NULL,
rejected INT NOT NULL,
reported INT NOT NULL,
"timestamp" TIMESTAMP WITHOUT TIME ZONE NOT NULL,
CONSTRAINT stat_reports_pkey PRIMARY KEY (id)
);
......@@ -5,7 +5,7 @@
<parent>
<groupId>fr.gouv.clea</groupId>
<artifactId>clea-server</artifactId>
<version>1.1.2</version>
<version>0-SNAPSHOT</version>
</parent>
<artifactId>clea-client</artifactId>
......
-- Needs: Clea-Statistiques
CREATE TABLE stat_reports
(
id VARCHAR(64) NOT NULL DEFAULT uuid_generate_v4(),
backwards INT NOT NULL,
is_closed INT NOT NULL,
forwards INT NOT NULL,
rejected INT NOT NULL,
reported INT NOT NULL,
dt_report TIMESTAMP WITHOUT TIME ZONE NOT NULL,
CONSTRAINT stat_reports_pkey PRIMARY KEY (id)
);
......@@ -4,7 +4,7 @@
<parent>
<groupId>fr.gouv.clea</groupId>
<artifactId>clea-server</artifactId>
<version>1.1.2</version>
<version>0-SNAPSHOT</version>
</parent>
<artifactId>clea-qr-simulator</artifactId>
......
......@@ -5,7 +5,7 @@
<parent>
<groupId>fr.gouv.clea</groupId>
<artifactId>clea-server</artifactId>
<version>1.1.2</version>
<version>0-SNAPSHOT</version>
</parent>
<artifactId>clea-scoring-conf</artifactId>
......
## CLEA Venue Consumer
Listener meant to retrieve decoded visits from the kafka queue anonymously, and calculating expositions.
### Verifications
For each record read from the kafka queue, the visit will be decrypted using the configured secret key and the following
verifications will be applied:
* if there's an error while decrypting a specific visit, it will be rejected.
* if
the [drift check]("https://gitlab.inria.fr/stopcovid19/CLEA-exposure-verification/-/blob/master/documents/CLEA-specification-EN.md#processing-of-a-user-location-record-by-the-backend-server")
fails, it will be rejected.
* if the temporary public if of the location, is different from the one calculated from the location specific part, it
will be rejected.
### Exposition slots calculation
After a visit is decrypted and verified, a combination of exposition slots will be generated and stored in DB.
for more details on the
specs https://gitlab.inria.fr/stopcovid19/backend-server/-/blob/clea-doc/documentation/clea-specs.MD
### Purge
Each day, at 01:00 PM UTC (configurable), a croned job will launch to purge all outdated entries from DB.
Outdated entries are processed following the retention date (14 days).
......@@ -6,7 +6,7 @@
<parent>
<groupId>fr.gouv.clea</groupId>
<artifactId>clea-server</artifactId>
<version>1.1.2</version>
<version>0-SNAPSHOT</version>
</parent>
<artifactId>clea-venue-consumer</artifactId>
......
package fr.gouv.clea.consumer.configuration;
import lombok.Data;
import lombok.NoArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import org.springframework.boot.context.properties.ConfigurationProperties;
import org.springframework.context.annotation.Configuration;
import org.springframework.validation.annotation.Validated;
import javax.validation.constraints.NotBlank;
@Data
@NoArgsConstructor
@Validated
@Configuration
@ConfigurationProperties(prefix = "clea.kafka")
@Slf4j
public class CleaKafkaProperties {
@NotBlank
private String qrCodesTopic;
@NotBlank
private String statsTopic;
}
package fr.gouv.clea.consumer.configuration;
import fr.gouv.clea.consumer.model.DecodedVisit;
import fr.gouv.clea.consumer.model.ReportStat;
import fr.gouv.clea.consumer.utils.KafkaVisitDeserializer;
import lombok.RequiredArgsConstructor;
import org.apache.kafka.clients.consumer.ConsumerConfig;
import org.apache.kafka.common.serialization.StringDeserializer;
import org.springframework.boot.autoconfigure.kafka.KafkaProperties;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
import org.springframework.kafka.annotation.EnableKafka;
import org.springframework.kafka.config.ConcurrentKafkaListenerContainerFactory;
import org.springframework.kafka.core.ConsumerFactory;
import org.springframework.kafka.core.DefaultKafkaConsumerFactory;
import org.springframework.kafka.support.serializer.JsonDeserializer;
import java.util.HashMap;
import java.util.Map;
@Configuration
@EnableKafka
@RequiredArgsConstructor
public class KafkaConfiguration {
private static final String OFFSET_CONFIG = "earliest";
private final KafkaProperties kafkaProperties;
@Bean
public ConsumerFactory<String, DecodedVisit> visitConsumerFactory() {
final var props = kafkaProperties.buildConsumerProperties();
props.put(ConsumerConfig.AUTO_OFFSET_RESET_CONFIG, OFFSET_CONFIG);
props.put(ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG, StringDeserializer.class);
props.put(ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG, KafkaVisitDeserializer.class);
props.put(JsonDeserializer.VALUE_DEFAULT_TYPE, DecodedVisit.class);
return new DefaultKafkaConsumerFactory<>(props);
}
@Bean
public ConcurrentKafkaListenerContainerFactory<String, DecodedVisit> visitContainerFactory() {
ConcurrentKafkaListenerContainerFactory<String, DecodedVisit> factory = new ConcurrentKafkaListenerContainerFactory<>();
factory.setConsumerFactory(visitConsumerFactory());
return factory;
}
@Bean
public ConsumerFactory<String, ReportStat> statConsumerFactory() {
final var props = kafkaProperties.buildConsumerProperties();
props.put(ConsumerConfig.AUTO_OFFSET_RESET_CONFIG, OFFSET_CONFIG);
props.put(ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG, StringDeserializer.class);
props.put(ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG, JsonDeserializer.class);
props.put(JsonDeserializer.VALUE_DEFAULT_TYPE, ReportStat.class);
return new DefaultKafkaConsumerFactory<>(props);
}
@Bean
public ConcurrentKafkaListenerContainerFactory<String, ReportStat> statContainerFactory() {
ConcurrentKafkaListenerContainerFactory<String, ReportStat> factory = new ConcurrentKafkaListenerContainerFactory<>();
factory.setConsumerFactory(statConsumerFactory());
return factory;
}
}
......@@ -2,25 +2,19 @@ package fr.gouv.clea.consumer.configuration;
import fr.inria.clea.lsp.CleaEciesEncoder;
import fr.inria.clea.lsp.LocationSpecificPartDecoder;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Value;
import lombok.RequiredArgsConstructor;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
@Configuration
@RequiredArgsConstructor
public class SecurityConfiguration {
private final String serverAuthoritySecretKey;
@Autowired
public SecurityConfiguration(
@Value("${clea.conf.security.crypto.serverAuthoritySecretKey}") String serverAuthoritySecretKey) {
this.serverAuthoritySecretKey = serverAuthoritySecretKey;
}
private final VenueConsumerProperties properties;
@Bean
public LocationSpecificPartDecoder getLocationSpecificPartDecoder() {
return new LocationSpecificPartDecoder(serverAuthoritySecretKey);
return new LocationSpecificPartDecoder(properties.getSecurity().getCrypto().getServerAuthoritySecretKey());
}
@Bean
......
......@@ -12,6 +12,8 @@ import org.springframework.validation.annotation.Validated;
import javax.annotation.PostConstruct;
import javax.validation.constraints.Max;
import javax.validation.constraints.Min;
import javax.validation.constraints.NotBlank;
import javax.validation.constraints.NotNull;
import javax.validation.constraints.Positive;
@Data
......@@ -22,21 +24,51 @@ import javax.validation.constraints.Positive;
@Configuration
@ConfigurationProperties(prefix = "clea.conf")
@Slf4j
public class VenueConsumerConfiguration {
public class VenueConsumerProperties {
@Min(value = 600)
private long durationUnitInSeconds;
@Min(value = 1800)
private long statSlotDurationInSeconds;
@Positive
private int driftBetweenDeviceAndOfficialTimeInSecs;
@Positive
private int cleaClockDriftInSecs;
@Min(value = 10)
@Max(value = 30)
private int retentionDurationInDays;
@NotNull
private Security security;
@PostConstruct
private void logConfiguration() {
log.info(this.toString());
}
}
@Data
@NoArgsConstructor
@AllArgsConstructor
@Builder
@Validated
class Security {
@NotNull
private Crypto crypto;
}
@Data
@NoArgsConstructor
@AllArgsConstructor
@Builder
@Validated
class Crypto {
@NotBlank
private String serverAuthoritySecretKey;
}
package fr.gouv.clea.consumer.model;
import fr.inria.clea.lsp.utils.TimeUtils;
import lombok.AllArgsConstructor;
import lombok.Builder;
import lombok.Data;
import lombok.NoArgsConstructor;
@Data
@NoArgsConstructor
@AllArgsConstructor
@Builder
public class ReportStat {
private int reported;
private int rejected;
private int backwards;
private int forwards;
private int close;
private long timestamp;
public ReportStatEntity toEntity() {
return ReportStatEntity.builder()
.reported(this.reported)
.rejected(this.rejected)
.backwards(this.backwards)
.forwards(this.forwards)
.close(this.close)
.timestamp(TimeUtils.instantFromTimestamp(this.timestamp))
.build();
}
}
package fr.gouv.clea.consumer.model;
import lombok.AllArgsConstructor;
import lombok.Builder;
import lombok.Data;
import lombok.NoArgsConstructor;
import org.hibernate.annotations.GenericGenerator;
import javax.persistence.Column;
import javax.persistence.Entity;
import javax.persistence.GeneratedValue;
import javax.persistence.Id;
import javax.persistence.Table;
import java.time.Instant;
@Data
@NoArgsConstructor
@AllArgsConstructor
@Builder
@Entity
@Table(name = "STAT_REPORTS")
public class ReportStatEntity {
@Id
@GeneratedValue(generator = "uuid")
@GenericGenerator(name = "uuid", strategy = "org.hibernate.id.UUIDGenerator")
private String id;
private int reported;
private int rejected;
private int backwards;
private int forwards;
@Column(name = "is_closed")
private int close;
@Column(name = "dt_report")
private Instant timestamp;
}
package fr.gouv.clea.consumer.repository;
import fr.gouv.clea.consumer.model.ReportStatEntity;
import org.springframework.data.jpa.repository.JpaRepository;
public interface IReportStatRepository extends JpaRepository<ReportStatEntity, Long> {
}
package fr.gouv.clea.consumer.service;
import fr.gouv.clea.consumer.model.DecodedVisit;
import fr.gouv.clea.consumer.model.ReportStat;
public interface IConsumerService {
void consume(DecodedVisit decodedVisit);
void consumeVisit(DecodedVisit decodedVisit);
void consumeStat(ReportStat reportStat);
}
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment