feat: add two reprocess endpoints

.github/workflows/docker.yaml

@@ -3,7 +3,8 @@ name: Build snapshot docker image
 on:
   push:
     branches-ignore:
-      - main
+      - * # main
+      # turned off for now (needs secrets renewal)
 
 jobs:
   build-snapshot:

CHANGELOG.md

@@ -1,4 +1,9 @@
 # Changelog
+
+## 2.4.7 [2026-04-10]
+### Added
+- Reprocess endpoints for raw responses with optional sinceDate and endDate filters
+
 ## 2.4.0 [2026-03-04]
 ### Added
 - Restful endpoints to get latest responses without the logic in it (not to be confused with get latest states)

src/main/java/fr/insee/genesis/controller/exception/ExceptionController.java

@@ -0,0 +1,44 @@
+package fr.insee.genesis.controller.exception;
+
+import fr.insee.genesis.exceptions.GenesisException;
+import fr.insee.genesis.exceptions.InvalidDateIntervalException;
+import lombok.extern.slf4j.Slf4j;
+import org.springframework.http.HttpStatus;
+import org.springframework.http.ProblemDetail;
+import org.springframework.web.bind.annotation.ControllerAdvice;
+import org.springframework.web.bind.annotation.ExceptionHandler;
+
+/**
+ * This controller uses Spring's ControllerAdvice annotation to intercept exceptions.
+ * It implements the <a href="https://www.rfc-editor.org/rfc/rfc9457.html">RFC 9457</a> by returning
+ * Spring's <code>ProblemDetail</code> object.
+ */
+@ControllerAdvice
+@Slf4j
+public class ExceptionController {
+
+    // Note: No handler for uncaught Exception.class for now since it breaks soms tests.
+
+    @ExceptionHandler
+    public ProblemDetail handleGenericGenesisException(GenesisException genesisException) {
+        log.error("GenesisException: {}", genesisException.getMessage(), genesisException);
+        return ProblemDetail.forStatusAndDetail(
+                resolveHttpCode(genesisException.getStatus()),
+                genesisException.getMessage());
+    }
+
+    /** Returns the corresponding http status, or 500 if the given code does not match a http status. */
+    private static HttpStatus resolveHttpCode(int statusCode) {
+        HttpStatus httpStatus = HttpStatus.resolve(statusCode);
+        return httpStatus != null ? httpStatus : HttpStatus.INTERNAL_SERVER_ERROR;
+    }
+
+    @ExceptionHandler(InvalidDateIntervalException.class)
+    public ProblemDetail handleInvalidDateIntervalException(InvalidDateIntervalException e) {
+        log.error("InvalidDateIntervalException: {}", e.getMessage());
+        return ProblemDetail.forStatusAndDetail(
+                HttpStatus.BAD_REQUEST,
+                e.getMessage());
+    }
+
+}

src/main/java/fr/insee/genesis/controller/rest/responses/InterrogationController.java

@@ -15,6 +15,7 @@
 import org.springframework.web.bind.annotation.RequestMapping;
 import org.springframework.web.bind.annotation.RequestParam;
 
+import java.time.Instant;
 import java.time.LocalDateTime;
 import java.util.List;
 
@@ -55,17 +56,17 @@ public ResponseEntity<List<InterrogationId>> getAllInterrogationIdsByCollectionI
             @DateTimeFormat(iso = DateTimeFormat.ISO.DATE_TIME)
             @Parameter(
                     description = "sinceDate",
-                    schema = @Schema(type = "string", format = "date-time", example = "2026-01-01T00:00:00")
+                    schema = @Schema(type = "string", format = "date-time", example = "2026-01-01T00:00:00Z")
             )
-            LocalDateTime start,
+            Instant start,
 
             @RequestParam("end")
             @DateTimeFormat(iso = DateTimeFormat.ISO.DATE_TIME)
             @Parameter(
                     description = "untilDate",
-                    schema = @Schema(type = "string", format = "date-time", example = "2026-01-31T23:59:59")
+                    schema = @Schema(type = "string", format = "date-time", example = "2026-01-31T23:59:59Z")
             )
-            LocalDateTime end) {
+            Instant end) {
         List<InterrogationId> responses = surveyUnitService.findDistinctInterrogationIdsByCollectionInstrumentIdAndRecordDateBetween(collectionInstrumentId, start,end);
         return ResponseEntity.ok(responses);
     }

src/main/java/fr/insee/genesis/controller/rest/responses/RawResponseController.java

@@ -15,6 +15,7 @@
 import io.swagger.v3.oas.annotations.Operation;
 import io.swagger.v3.oas.annotations.Parameter;
 import jakarta.validation.Valid;
+import lombok.RequiredArgsConstructor;
 import lombok.extern.slf4j.Slf4j;
 import org.springframework.data.domain.Page;
 import org.springframework.data.domain.PageRequest;
@@ -43,23 +44,16 @@
 
 @Slf4j
 @Controller
+@RequiredArgsConstructor
 public class RawResponseController {
 
     private static final String SUCCESS_MESSAGE = "Interrogation %s saved";
     private static final String INTERROGATION_ID = "interrogationId";
-    public static final String NB_DOCS_WITH_FORMATTED = "%d document(s) processed, including %d FORMATTED after data verification for collectionInstrumentId %s";
-    public static final String NB_DOCS = "%d document(s) processed for collectionInstrumentId %s";
+
     private final LunaticJsonRawDataApiPort lunaticJsonRawDataApiPort;
     private final RawResponseApiPort rawResponseApiPort;
     private final RawResponseInputRepository rawRepository;
 
-
-    public RawResponseController(LunaticJsonRawDataApiPort lunaticJsonRawDataApiPort, RawResponseApiPort rawResponseApiPort, RawResponseInputRepository rawRepository) {
-        this.lunaticJsonRawDataApiPort = lunaticJsonRawDataApiPort;
-        this.rawResponseApiPort = rawResponseApiPort;
-        this.rawRepository = rawRepository;
-    }
-
     @Operation(summary = "Save lunatic json data from one interrogation in Genesis Database")
     @PutMapping(path = "/responses/raw/lunatic-json/save")
     @PreAuthorize("hasRole('COLLECT_PLATFORM')")
@@ -118,11 +112,8 @@ public ResponseEntity<String> processRawResponses(
         log.info("Try to process raw responses for collectionInstrumentId {} and {} interrogationIds", collectionInstrumentId, interrogationIdList.size());
         List<GenesisError> errors = new ArrayList<>();
         try {
-            DataProcessResult result = rawResponseApiPort.processRawResponses(collectionInstrumentId, interrogationIdList, errors);
-            return result.formattedDataCount() == 0 ?
-                    ResponseEntity.ok(NB_DOCS.formatted(result.dataCount(), collectionInstrumentId))
-                    : ResponseEntity.ok(NB_DOCS_WITH_FORMATTED
-                    .formatted(result.dataCount(), result.formattedDataCount(), collectionInstrumentId));
+            DataProcessResult result = rawResponseApiPort.processRawResponsesByInterrogationIds(collectionInstrumentId, interrogationIdList, errors);
+            return ResponseEntity.ok(result.message(collectionInstrumentId));
         } catch (GenesisException e) {
             return ResponseEntity.status(e.getStatus()).body(e.getMessage());
         }
@@ -140,11 +131,8 @@ public ResponseEntity<String> processRawResponsesByCollectionInstrumentId(
     ) {
         log.info("Try to process raw responses for collectionInstrumentId {}", collectionInstrumentId);
         try {
-            DataProcessResult result = rawResponseApiPort.processRawResponses(collectionInstrumentId);
-            return result.formattedDataCount() == 0 ?
-                    ResponseEntity.ok(NB_DOCS.formatted(result.dataCount(), collectionInstrumentId))
-                    : ResponseEntity.ok(NB_DOCS_WITH_FORMATTED
-                    .formatted(result.dataCount(), result.formattedDataCount(), collectionInstrumentId));
+            DataProcessResult result = rawResponseApiPort.processRawResponsesByInterrogationIds(collectionInstrumentId);
+            return ResponseEntity.ok(result.message(collectionInstrumentId));
         } catch (GenesisException e) {
             return ResponseEntity.status(e.getStatus()).body(e.getMessage());
         }
@@ -183,7 +171,7 @@ public ResponseEntity<LunaticJsonRawDataModel> getJsonRawData(
             @RequestParam("campaignName") String campaignName,
             @RequestParam(value = "mode") Mode modeSpecified
     ) {
-        List<LunaticJsonRawDataModel> data = lunaticJsonRawDataApiPort.getRawData(campaignName, modeSpecified, List.of(interrogationId));
+        List<LunaticJsonRawDataModel> data = lunaticJsonRawDataApiPort.getRawDataByQuestionnaireId(campaignName, modeSpecified, List.of(interrogationId));
         return ResponseEntity.ok(data.getFirst());
     }
 
@@ -201,7 +189,7 @@ public ResponseEntity<String> processJsonRawData(
         List<GenesisError> errors = new ArrayList<>();
 
         try {
-            DataProcessResult result = lunaticJsonRawDataApiPort.processRawData(campaignName, interrogationIdList, errors);
+            DataProcessResult result = lunaticJsonRawDataApiPort.processRawDataByInterrogationIds(campaignName, interrogationIdList, errors);
             return result.formattedDataCount() == 0 ?
                     ResponseEntity.ok("%d document(s) processed".formatted(result.dataCount()))
                     : ResponseEntity.ok("%d document(s) processed, including %d FORMATTED after data verification"

src/main/java/fr/insee/genesis/controller/rest/responses/RawResponseReprocessController.java

@@ -0,0 +1,102 @@
+package fr.insee.genesis.controller.rest.responses;
+
+import fr.insee.genesis.domain.model.surveyunit.rawdata.DataProcessResult;
+import fr.insee.genesis.domain.model.surveyunit.rawdata.RawDataModelType;
+import fr.insee.genesis.domain.ports.api.ReprocessRawResponseApiPort;
+import fr.insee.genesis.exceptions.GenesisException;
+import io.swagger.v3.oas.annotations.Operation;
+import io.swagger.v3.oas.annotations.Parameter;
+import io.swagger.v3.oas.annotations.media.Schema;
+import lombok.RequiredArgsConstructor;
+import lombok.extern.slf4j.Slf4j;
+import org.springframework.format.annotation.DateTimeFormat;
+import org.springframework.http.ResponseEntity;
+import org.springframework.security.access.prepost.PreAuthorize;
+import org.springframework.stereotype.Controller;
+import org.springframework.web.bind.annotation.PathVariable;
+import org.springframework.web.bind.annotation.PostMapping;
+import org.springframework.web.bind.annotation.RequestParam;
+
+import java.time.Instant;
+
+@Controller
+@RequiredArgsConstructor
+@Slf4j
+public class RawResponseReprocessController {
+
+    private final ReprocessRawResponseApiPort reprocessRawResponseApiPort;
+
+    @Operation(summary = "Reprocess raw response of a collection instrument.")
+    @PostMapping(path = "/raw-responses/{collectionInstrumentId}/reprocess")
+    @PreAuthorize("hasRole('ADMIN')")
+    public ResponseEntity<String> reProcessRawResponsesByCollectionInstrumentId(
+            @Parameter(
+                    description = "Id of the collection instrument (old questionnaireId)",
+                    example = "ENQTEST2025X00")
+            @PathVariable("collectionInstrumentId")
+            String collectionInstrumentId,
+
+            @Parameter(
+                    description = "Extract since",
+                    schema = @Schema(type = "string", format = "date-time", example = "2026-01-01T00:00:00Z")
+            )
+            @RequestParam(value = "sinceDate", required = false)
+            @DateTimeFormat(iso = DateTimeFormat.ISO.DATE_TIME)
+            Instant sinceDate,
+
+            @Parameter(
+                    description = "Extract until",
+                    schema = @Schema(type = "string", format = "date-time", example = "2026-02-02T00:00:00Z")
+            )
+            @RequestParam(value = "endDate", required = false)
+            @DateTimeFormat(iso = DateTimeFormat.ISO.DATE_TIME)
+            Instant endDate
+    ) throws GenesisException {
+
+        DataProcessResult result = reprocessRawResponseApiPort.reprocessRawResponses(
+                RawDataModelType.FILIERE,
+                collectionInstrumentId,
+                sinceDate,
+                endDate);
+
+        return ResponseEntity.ok(result.message(collectionInstrumentId));
+    }
+
+    @Operation(summary = "Reprocess Lunatic raw data for a questionnaire model. " +
+            "**Note**: Lunatic raw data is the legacy format of raw responses.")
+    @PostMapping(path = "/responses/raw/lunatic-json/{questionnaireId}/reprocess")
+    @PreAuthorize("hasRole('ADMIN')")
+    public ResponseEntity<String> reProcessJsonRawDataByQuestionnaireId(
+            @Parameter(
+                    description = "Questionnaire model id (old name for collection instrument id).",
+                    example = "ENQTEST2025X00")
+            @PathVariable("questionnaireId")
+            String collectionInstrumentId, // 'questionnaireId' is the legacy name for 'collectionInstrumentId'
+
+            @Parameter(
+                    description = "Extract since",
+                    schema = @Schema(type = "string", format = "date-time", example = "2026-01-01T00:00:00Z")
+            )
+            @RequestParam(value = "sinceDate", required = false)
+            @DateTimeFormat(iso = DateTimeFormat.ISO.DATE_TIME)
+            Instant sinceDate,
+
+            @Parameter(
+                    description = "Extract until",
+                    schema = @Schema(type = "string", format = "date-time", example = "2026-02-02T00:00:00Z")
+            )
+            @RequestParam(value = "endDate", required = false)
+            @DateTimeFormat(iso = DateTimeFormat.ISO.DATE_TIME)
+            Instant endDate
+    ) throws GenesisException {
+
+        DataProcessResult result = reprocessRawResponseApiPort.reprocessRawResponses(
+                RawDataModelType.LEGACY,
+                collectionInstrumentId,
+                sinceDate,
+                endDate);
+
+        return ResponseEntity.ok(result.message(collectionInstrumentId));
+    }
+
+}

src/main/java/fr/insee/genesis/controller/utils/ControllerUtils.java

@@ -26,19 +26,19 @@ public ControllerUtils(FileUtils fileUtils) {
 
 	/**
 	 * If a mode is specified, we treat only this mode.
-	 * If no mode is specified, we treat all modes in the campaign.
+	 * If no mode is specified, we treat all modes in the questionnaireId.
 	 * If no mode is specified and no specs are found, we return an error
-	 * @param campaign campaign id to get modes
+	 * @param questionnaireId questionnaireId id to get modes
 	 * @param modeSpecified a Mode to use, null if we want all modes available
 	 * @return a list with the mode in modeSpecified or all modes if null
 	 * @throws GenesisException if error in specs structure
 	 */
-	public List<Mode> getModesList(String campaign, Mode modeSpecified) throws GenesisException {
+	public List<Mode> getModesList(String questionnaireId, Mode modeSpecified) throws GenesisException {
 		if (modeSpecified != null){
 			return Collections.singletonList(modeSpecified);
 		}
 		List<Mode> modes = new ArrayList<>();
-		String specFolder = fileUtils.getSpecFolder(campaign);
+		String specFolder = fileUtils.getSpecFolder(questionnaireId);
 		List<String> modeSpecFolders = fileUtils.listFolders(specFolder);
 		if (modeSpecFolders.isEmpty()) {
 			throw new GenesisException(404, "No specification folder found " + specFolder);

src/main/java/fr/insee/genesis/domain/model/surveyunit/rawdata/DataProcessResult.java

@@ -4,5 +4,31 @@
 
 import java.util.List;
 
-public record DataProcessResult(int dataCount, int formattedDataCount, List<GenesisError> errors) {
+public record DataProcessResult(
+        int dataCount,
+        int formattedDataCount,
+        List<GenesisError> errors) {
+
+    public String message(String collectionInstrumentId) {
+        return String.format("%s%s%s.",
+                interrogationCountMessage(dataCount),
+                formattedCountMessage(formattedDataCount),
+                collectionIdMessage(collectionInstrumentId));
+    }
+
+    private static String interrogationCountMessage(int processedInterrogationsCount) {
+        boolean plural = processedInterrogationsCount > 1;
+        return "%d interrogation%s processed".formatted(processedInterrogationsCount,  plural ? "s" : "");
+    }
+
+    private static String collectionIdMessage(String collectionInstrumentId) {
+        return " for collectionInstrumentId '%s'".formatted(collectionInstrumentId);
+    }
+
+    private static String formattedCountMessage(int formattedCount) {
+        if (formattedCount == 0)
+            return "";
+        return " (including %d FORMATTED after data verification)".formatted(formattedCount);
+    }
+
 }

src/main/java/fr/insee/genesis/domain/model/surveyunit/rawdata/RawDataModelType.java

@@ -1,5 +1,20 @@
 package fr.insee.genesis.domain.model.surveyunit.rawdata;
 
+/** Format of raw data to be imported into the data storage. */
 public enum RawDataModelType {
-    DEFAULT, FILIERE
+
+    /** Legacy format of raw data ('Lunatic'). */
+    LEGACY,
+
+    /** 'Filière' raw response model. */
+    FILIERE;
+
+    @Override
+    public String toString() {
+        return switch (this) {
+            case LEGACY -> "LEGACY (Lunatic)";
+            case FILIERE -> "FILIERE raw responses";
+        };
+    }
+
 }

src/main/java/fr/insee/genesis/domain/ports/api/LunaticJsonRawDataApiPort.java

@@ -11,18 +11,18 @@
 import org.springframework.data.domain.Page;
 import org.springframework.data.domain.Pageable;
 
-import java.time.LocalDateTime;
 import java.time.Instant;
+import java.time.LocalDateTime;
 import java.util.List;
 import java.util.Map;
 import java.util.Set;
 
 public interface LunaticJsonRawDataApiPort {
 
     void save(LunaticJsonRawDataModel rawData);
-    List<LunaticJsonRawDataModel> getRawData(String campaignName, Mode mode, List<String> interrogationIdList);
     List<LunaticJsonRawDataModel> getRawDataByQuestionnaireId(String questionnaireId, Mode mode, List<String> interrogationIdList);
     List<SurveyUnitModel> convertRawData(List<LunaticJsonRawDataModel> rawData, VariablesMap variablesMap);
+
     List<LunaticJsonRawDataUnprocessedDto> getUnprocessedDataIds();
     Set<String> getUnprocessedDataQuestionnaireIds();
     void updateProcessDates(List<SurveyUnitModel> surveyUnitModels);
@@ -33,8 +33,10 @@
 
     List<LunaticJsonRawDataModel> getRawDataByInterrogationId(String interrogationId);
 
-    @Deprecated(since = "1.13.0")
-    DataProcessResult processRawData(String campaignName, List<String> interrogationIdList, List<GenesisError> errors) throws GenesisException;
+    /**
+     * @deprecated Use the method with 'collectionInstrumentId' instead.
+     */
+    DataProcessResult processRawDataByInterrogationIds(String campaignName, List<String> interrogationIdList, List<GenesisError> errors) throws GenesisException;
 
     DataProcessResult processRawData(String collectionInstrumentId) throws GenesisException;