Improve differential extraction (interrogation and last extraction date endpoints)

src/main/java/fr/insee/genesis/controller/adapter/LunaticJsonAdapter.java

@@ -6,7 +6,7 @@
 import fr.insee.genesis.domain.model.surveyunit.SurveyUnitModel;
 import org.springframework.stereotype.Service;
 
-import java.time.LocalDateTime;
+import java.time.Instant;
 
 @Service
 public class LunaticJsonAdapter {
@@ -18,7 +18,7 @@ public SurveyUnitModel convert(LunaticJsonSurveyUnit su){
                 .interrogationId(su.getInterrogationId())
                 .state(DataState.COLLECTED)
                 .mode(Mode.WEB)
-                .recordDate(LocalDateTime.now())
+                .recordDate(Instant.now())
                 .build();
     }
 

src/main/java/fr/insee/genesis/controller/adapter/LunaticXmlAdapter.java

@@ -5,14 +5,14 @@
 import fr.insee.genesis.controller.sources.xml.LunaticXmlOtherData;
 import fr.insee.genesis.controller.sources.xml.LunaticXmlSurveyUnit;
 import fr.insee.genesis.controller.sources.xml.ValueType;
-import fr.insee.genesis.domain.model.surveyunit.SurveyUnitModel;
-import fr.insee.genesis.domain.utils.GroupUtils;
 import fr.insee.genesis.domain.model.surveyunit.DataState;
 import fr.insee.genesis.domain.model.surveyunit.Mode;
+import fr.insee.genesis.domain.model.surveyunit.SurveyUnitModel;
 import fr.insee.genesis.domain.model.surveyunit.VariableModel;
+import fr.insee.genesis.domain.utils.GroupUtils;
 import lombok.experimental.UtilityClass;
 
-import java.time.LocalDateTime;
+import java.time.Instant;
 import java.util.ArrayList;
 import java.util.List;
 
@@ -74,7 +74,7 @@ private static SurveyUnitModel getStateDataFromSurveyUnit(LunaticXmlSurveyUnit s
                 .interrogationId(su.getId())
                 .state(dataState)
                 .mode(mode)
-                .recordDate(LocalDateTime.now())
+                .recordDate(Instant.now())
                 .fileDate(su.getFileDate())
                 .build();
 

src/main/java/fr/insee/genesis/controller/dto/InterrogationBatchResponse.java

@@ -0,0 +1,16 @@
+package fr.insee.genesis.controller.dto;
+
+import fr.insee.genesis.domain.model.surveyunit.InterrogationId;
+import lombok.Data;
+
+import java.time.Instant;
+import java.util.ArrayList;
+import java.util.List;
+
+@Data
+public class InterrogationBatchResponse {
+
+    private List<InterrogationId> interrogationIds = new ArrayList<>();
+    private Instant nextSince;
+
+}

src/main/java/fr/insee/genesis/controller/dto/LastExtractionRequest.java

@@ -0,0 +1,14 @@
+package fr.insee.genesis.controller.dto;
+
+import jakarta.validation.constraints.NotNull;
+import lombok.Data;
+
+import java.time.Instant;
+
+@Data
+public class LastExtractionRequest {
+
+    @NotNull
+    private Instant lastExtractionDate;
+
+}

src/main/java/fr/insee/genesis/controller/dto/LastExtractionResponseDto.java

@@ -3,13 +3,13 @@
 import lombok.Getter;
 import lombok.Setter;
 
-import java.time.LocalDateTime;
+import java.time.Instant;
 
 @Getter
 @Setter
 public class LastExtractionResponseDto {
     private final String lastExtractionDate;
-    public LastExtractionResponseDto(LocalDateTime lastExtractionDate) {
+    public LastExtractionResponseDto(Instant lastExtractionDate) {
         this.lastExtractionDate = lastExtractionDate != null ? lastExtractionDate.toString() : null;
     }
 }

src/main/java/fr/insee/genesis/controller/dto/VariableStateDto.java

@@ -1,19 +1,16 @@
 package fr.insee.genesis.controller.dto;
 
-import com.fasterxml.jackson.annotation.JsonFormat;
 import fr.insee.genesis.domain.model.surveyunit.DataState;
 import lombok.Builder;
 import lombok.Data;
 
-import java.time.LocalDateTime;
+import java.time.Instant;
 
 @Builder
 @Data
 public class VariableStateDto {
     private DataState state;
     private boolean active;
     private Object value;
-
-    @JsonFormat(shape = JsonFormat.Shape.STRING, pattern = "dd/MM/yyyy HH:mm:ss.SSS")
-    private LocalDateTime date;
+    private Instant date;
 }

src/main/java/fr/insee/genesis/controller/rest/JsonExtractionController.java

@@ -1,53 +1,59 @@
 package fr.insee.genesis.controller.rest;
 
+import fr.insee.genesis.controller.dto.LastExtractionRequest;
 import fr.insee.genesis.controller.dto.LastExtractionResponseDto;
 import fr.insee.genesis.domain.model.extraction.json.LastJsonExtractionModel;
 import fr.insee.genesis.domain.model.surveyunit.Mode;
 import fr.insee.genesis.domain.ports.api.LastJsonExtractionApiPort;
 import fr.insee.genesis.exceptions.GenesisException;
 import io.swagger.v3.oas.annotations.Operation;
+import jakarta.validation.Valid;
 import lombok.AllArgsConstructor;
 import lombok.extern.slf4j.Slf4j;
 import org.springframework.http.ResponseEntity;
 import org.springframework.security.access.prepost.PreAuthorize;
 import org.springframework.stereotype.Controller;
 import org.springframework.web.bind.annotation.DeleteMapping;
 import org.springframework.web.bind.annotation.GetMapping;
+import org.springframework.web.bind.annotation.PathVariable;
 import org.springframework.web.bind.annotation.PutMapping;
+import org.springframework.web.bind.annotation.RequestBody;
 import org.springframework.web.bind.annotation.RequestMapping;
 import org.springframework.web.bind.annotation.RequestParam;
 
-import java.time.LocalDateTime;
-
 @Slf4j
 @Controller
 @AllArgsConstructor
-@RequestMapping(path = "/extractions")
+@RequestMapping(path = "/collection-instruments")
 public class JsonExtractionController {
 
     LastJsonExtractionApiPort lastJsonExtractionApiPort;
 
     @Operation(summary = "Record the date of the latest JSON data extraction in Kraftwerk")
-    @PutMapping(path = "/json")
+    @PutMapping(path = "/{collectionInstrumentId}/extractions/json/last")
     @PreAuthorize("hasAnyRole('USER_KRAFTWERK','SCHEDULER')")
     public ResponseEntity<String> saveLastJsonExtractionDate(
-            @RequestParam("collectionInstrumentId") String collectionInstrumentId,
-            @RequestParam(value = "mode", required = false) Mode mode){
-        LocalDateTime extractDate = LocalDateTime.now();
+            @PathVariable String collectionInstrumentId,
+            @RequestParam(value = "mode", required = false) Mode mode,
+            @RequestBody @Valid LastExtractionRequest request){
         LastJsonExtractionModel extract = LastJsonExtractionModel.builder()
                 .collectionInstrumentId(collectionInstrumentId)
                 .mode(mode)
-                .lastExtractionDate(extractDate)
+                .lastExtractionDate(request.getLastExtractionDate())
                 .build();
+
         lastJsonExtractionApiPort.recordDate(extract);
+
         return ResponseEntity.ok().build();
     }
 
+
+
     @Operation(summary = "Get the date of the latest JSON data extraction in Kraftwerk")
-    @GetMapping(path = "/json")
+    @GetMapping(path = "/{collectionInstrumentId}/extractions/json/last")
     @PreAuthorize("hasAnyRole('USER_KRAFTWERK','SCHEDULER')")
     public ResponseEntity<LastExtractionResponseDto> getLastJsonExtractionDate(
-            @RequestParam("collectionInstrumentId") String collectionInstrumentId,
+            @PathVariable String collectionInstrumentId,
             @RequestParam(value = "mode", required = false) Mode mode){
         try{
             LastJsonExtractionModel lastJsonExtraction = lastJsonExtractionApiPort.getLastExtractionDate(collectionInstrumentId,mode);
@@ -58,10 +64,10 @@ public ResponseEntity<LastExtractionResponseDto> getLastJsonExtractionDate(
     }
 
     @Operation(summary = "Reset latest JSON data extraction")
-    @DeleteMapping(path = "/json")
+    @DeleteMapping(path = "/{collectionInstrumentId}/extractions/json/last")
     @PreAuthorize("hasRole('ADMIN')")
     public ResponseEntity<Object> deleteJsonExtractionDate(
-            @RequestParam("collectionInstrumentId") String collectionInstrumentId,
+            @PathVariable String collectionInstrumentId,
             @RequestParam(value = "mode", required = false) Mode mode){
         try {
             lastJsonExtractionApiPort.delete(collectionInstrumentId, mode);

src/main/java/fr/insee/genesis/controller/rest/responses/InterrogationController.java

@@ -1,25 +1,28 @@
 package fr.insee.genesis.controller.rest.responses;
 
+import fr.insee.genesis.controller.dto.InterrogationBatchResponse;
 import fr.insee.genesis.controller.rest.CommonApiResponse;
 import fr.insee.genesis.domain.model.surveyunit.InterrogationId;
+import fr.insee.genesis.domain.model.surveyunit.InterrogationInfo;
 import fr.insee.genesis.domain.ports.api.SurveyUnitApiPort;
 import io.swagger.v3.oas.annotations.Operation;
 import io.swagger.v3.oas.annotations.Parameter;
 import io.swagger.v3.oas.annotations.media.Schema;
 import lombok.extern.slf4j.Slf4j;
+import org.jspecify.annotations.NonNull;
 import org.springframework.format.annotation.DateTimeFormat;
 import org.springframework.http.ResponseEntity;
 import org.springframework.stereotype.Controller;
 import org.springframework.web.bind.annotation.GetMapping;
 import org.springframework.web.bind.annotation.PathVariable;
-import org.springframework.web.bind.annotation.RequestMapping;
 import org.springframework.web.bind.annotation.RequestParam;
 
 import java.time.Instant;
 import java.time.LocalDateTime;
+import java.util.Comparator;
 import java.util.List;
+import java.util.Optional;
 
-@RequestMapping(path = "/interrogations" )
 @Controller
 @Slf4j
 public class InterrogationController implements CommonApiResponse {
@@ -33,23 +36,49 @@ public InterrogationController(SurveyUnitApiPort surveyUnitService) {
 
 
     @Operation(summary = "Retrieve all interrogations for a given questionnaire")
-    @GetMapping(path = "/by-questionnaire")
+    @GetMapping(path = "interrogations/by-questionnaire")
     public ResponseEntity<List<InterrogationId>> getAllInterrogationIdsByQuestionnaire(@RequestParam("questionnaireId") String questionnaireId) {
         List<InterrogationId> responses = surveyUnitService.findDistinctInterrogationIdsByQuestionnaireId(questionnaireId);
         return ResponseEntity.ok(responses);
     }
 
+    @Operation(summary = "Retrieve all interrogations for a given collection instrument")
+    @GetMapping(path = "collection-instruments/{collectionInstrumentId}/interrogations/all")
+    public ResponseEntity<InterrogationBatchResponse> getAllInterrogationIdsByCollectionInstrumentId(
+            @PathVariable String collectionInstrumentId) {
+        List<InterrogationInfo> idsInfo = surveyUnitService.findDistinctInterrogationIdsByCollectionInstrumentId(collectionInstrumentId);
+        InterrogationBatchResponse response = buildInterrogationBatchResponse(idsInfo);
+        return ResponseEntity.ok(response);
+    }
+
     @Operation(summary = "Retrieve interrogations recorded since a specified date for a given questionnaire")
-    @GetMapping(path = "/by-questionnaire-and-since-datetime")
-    public ResponseEntity<List<InterrogationId>> getAllInterrogationIdsByQuestionnaire(
-            @RequestParam("questionnaireId") String questionnaireId,
-            @DateTimeFormat(iso = DateTimeFormat.ISO.DATE_TIME) LocalDateTime since) {
-        List<InterrogationId> responses = surveyUnitService.findDistinctInterrogationIdsByQuestionnaireIdAndDateAfter(questionnaireId, since);
-        return ResponseEntity.ok(responses);
+    @GetMapping(path = "collection-instruments/{collectionInstrumentId}/interrogations")
+    public ResponseEntity<InterrogationBatchResponse> getAllInterrogationIdsByQuestionnaire(
+            @PathVariable String collectionInstrumentId,
+            @DateTimeFormat(iso = DateTimeFormat.ISO.DATE_TIME) Instant since) {
+        List<InterrogationInfo> idsInfo = surveyUnitService.findDistinctInterrogationIdsByCollectionInstrumentIdAndSince(collectionInstrumentId, since);
+        InterrogationBatchResponse response = buildInterrogationBatchResponse(idsInfo);
+        return ResponseEntity.ok(response);
+    }
+
+    private static @NonNull InterrogationBatchResponse buildInterrogationBatchResponse(List<InterrogationInfo> ids) {
+        Optional<Instant> maxTimeStamp = ids.stream()
+                .map(InterrogationInfo::recordDate)
+                .max(Comparator.naturalOrder());
+        InterrogationBatchResponse response = new InterrogationBatchResponse();
+        if (maxTimeStamp.isPresent()){
+            response.setInterrogationIds(ids.stream()
+                    .map(InterrogationInfo::interrogationId)
+                    .distinct()
+                    .map(InterrogationId::new)
+                    .toList());
+            response.setNextSince(maxTimeStamp.get());
+        }
+        return response;
     }
 
     @Operation(summary = "Retrieve interrogations recorded between two dates for a given collection instrument")
-    @GetMapping(path = "/by-collection-instrument-and-between-datetime")
+    @GetMapping(path = "interrogations/by-collection-instrument-and-between-datetime")
     public ResponseEntity<List<InterrogationId>> getAllInterrogationIdsByCollectionInstrumentIdAndDate(
             @RequestParam("collectionInstrumentId") String collectionInstrumentId,
             @RequestParam("start")
@@ -79,7 +108,7 @@ public ResponseEntity<List<InterrogationId>> getAllInterrogationIdsByCollectionI
      * @author Alexis Szmundy
      */
     @Operation(summary = "Retrieve number of interrogations for a given questionnaire/collection instrument")
-    @GetMapping(path = "/by-questionnaire/{questionnaireId}/count")
+    @GetMapping(path = "interrogations/by-questionnaire/{questionnaireId}/count")
     public ResponseEntity<Long> countAllInterrogationIdsByQuestionnaireOrCollectionInstrument(
             @Parameter(description = "questionnaireId/collectionInstrumentId", required = true) @PathVariable("questionnaireId") String questionnaireId
     ) {
@@ -93,7 +122,7 @@ public ResponseEntity<Long> countAllInterrogationIdsByQuestionnaireOrCollectionI
      * @author Adrien Marchal
      */
     @Operation(summary = "Retrieve paginated interrogations for a given questionnaire")
-    @GetMapping(path = "/by-questionnaire/{questionnaireId}/paginated")
+    @GetMapping(path = "interrogations/by-questionnaire/{questionnaireId}/paginated")
     public ResponseEntity<List<InterrogationId>> getPaginatedInterrogationIdsByQuestionnaire(
             @Parameter(description = "questionnaireId", required = true) @PathVariable("questionnaireId") String questionnaireId,
             @Parameter(description = "if totalSize is 0, a count query is made to get the real totalSize to process", required = false) @RequestParam(defaultValue = "0") long totalSize,

src/main/java/fr/insee/genesis/controller/rest/responses/ResponseController.java

@@ -34,6 +34,7 @@
 import fr.insee.genesis.infrastructure.utils.FileUtils;
 import fr.insee.modelefiliere.RawResponseDto;
 import io.swagger.v3.oas.annotations.Operation;
+import io.swagger.v3.oas.annotations.Parameter;
 import io.swagger.v3.oas.annotations.tags.Tag;
 import lombok.extern.slf4j.Slf4j;
 import org.springframework.http.HttpStatus;
@@ -63,6 +64,7 @@
 import java.nio.file.Files;
 import java.nio.file.Path;
 import java.nio.file.Paths;
+import java.time.Instant;
 import java.time.LocalDateTime;
 import java.util.ArrayList;
 import java.util.List;
@@ -333,10 +335,11 @@ public ResponseEntity<SurveyUnitSimplifiedDto> getResponseByCollectionInstrument
             @PathVariable("interrogationId") String interrogationId,
             @RequestParam("mode") Mode mode) {
             return ResponseEntity.ok(
-                    surveyUnitService.findSimplifiedByCollectionInstrumentIdAndInterrogationId(
+                    surveyUnitService.findSimplified(
                             collectionInstrumentId,
                             interrogationId,
-                            mode
+                            mode,
+                            null
                     )
             );
     }
@@ -401,19 +404,26 @@ public ResponseEntity<List<SurveyUnitSimplifiedDto>> getLatestForInterrogationLi
         return ResponseEntity.ok(results);
     }
 
-    @Operation(summary = "Retrieve all responses for a collection instrument and a list of interrogations",
+    @Operation(summary = "Retrieve responses for a collection instrument and a list of interrogations",
             description = "Return the latest state for each variable for the given interrogationIds and a given collection instrument (formerly questionnaire).<br>" +
-                    "For a given id, the endpoint returns a document by collection mode (if there is more than one).")
+                    "For a given id, the endpoint returns a document by collection mode (if there is more than one)<br>" +
+                    "If the 'recordedBefore' parameter is provided, only responses recorded before this timestamp will be returned.")
     @PostMapping(path = "/{collectionInstrumentId}")
     @PreAuthorize("hasRole('USER_KRAFTWERK')")
-    public ResponseEntity<List<SurveyUnitSimplifiedDto>> getResponseByCollectionInstrumentAndInterrogationList(
+    public ResponseEntity<List<SurveyUnitSimplifiedDto>> searchResponses(
             @PathVariable("collectionInstrumentId") String collectionInstrumentId,
+            @Parameter(
+                    description = "Filter responses to those recorded strictly before the given timestamp (ISO-8601 UTC format).",
+                    example = "2026-01-15T10:15:30Z"
+            )
+            @RequestParam(value = "recordedBefore", required = false) Instant recordedBefore,
             @RequestBody List<InterrogationId> interrogationIds)
     {
         return ResponseEntity.ok(
-                surveyUnitService.findSimplifiedByCollectionInstrumentIdAndInterrogationIdList(
+                surveyUnitService.findSimplifiedList(
                         collectionInstrumentId,
-                        interrogationIds
+                        interrogationIds,
+                        recordedBefore
                 )
         );
     }

src/main/java/fr/insee/genesis/domain/model/extraction/json/LastJsonExtractionModel.java

@@ -7,7 +7,7 @@
 import lombok.NoArgsConstructor;
 import org.springframework.data.annotation.Id;
 
-import java.time.LocalDateTime;
+import java.time.Instant;
 
 @Data
 @Builder
@@ -18,5 +18,5 @@ public class LastJsonExtractionModel {
         private String id; //Used to remove warning
         String collectionInstrumentId;
         Mode mode;
-        LocalDateTime lastExtractionDate;
+        Instant lastExtractionDate;
 }

src/main/java/fr/insee/genesis/domain/model/surveyunit/InterrogationInfo.java

@@ -0,0 +1,8 @@
+package fr.insee.genesis.domain.model.surveyunit;
+
+import java.time.Instant;
+
+public record InterrogationInfo(
+        String interrogationId,
+        Instant recordDate
+) {}