Merge remote-tracking branch 'upstream/main'

Author: ozgliderpilot

memex/ClassDescription.md

@@ -3,7 +3,7 @@
 import com.fasterxml.jackson.core.JsonProcessingException;
 import com.fasterxml.jackson.databind.ObjectMapper;
 import com.johnlpage.memex.dto.PageDto;
-import com.johnlpage.memex.model.UpdateStrategy;
+import com.johnlpage.memex.util.UpdateStrategy;
 import com.johnlpage.memex.model.Vehicle;
 import com.johnlpage.memex.model.VehicleInspection;
 import com.johnlpage.memex.repository.VehicleInspectionRepository;
@@ -164,9 +164,10 @@ public ResponseEntity<String> atlasSearchQuery(@RequestBody String requestBody)
   public ResponseEntity<StreamingResponseBody> streamJson() {
 
     return ResponseEntity.ok()
-            .contentType(MediaType.APPLICATION_JSON)
-            .body(outputStream ->
-                    writeDocumentsToOutputStream(outputStream, downstreamService.jsonExtractStream()));
+        .contentType(MediaType.APPLICATION_JSON)
+        .body(
+            outputStream ->
+                writeDocumentsToOutputStream(outputStream, downstreamService.jsonExtractStream()));
   }
 
   /**
@@ -195,10 +196,13 @@ public ResponseEntity<StreamingResponseBody> streamJsonNative() {
         """;
 
     return ResponseEntity.ok()
-            .contentType(MediaType.APPLICATION_JSON)
-            .body(outputStream -> {
-              try (BufferedOutputStream bufferedOutputStream = new BufferedOutputStream(outputStream);
-                   Stream<JsonObject> stream = downstreamService.nativeJsonExtractStream(formatRequired)) {
+        .contentType(MediaType.APPLICATION_JSON)
+        .body(
+            outputStream -> {
+              try (BufferedOutputStream bufferedOutputStream =
+                      new BufferedOutputStream(outputStream);
+                  Stream<JsonObject> stream =
+                      downstreamService.nativeJsonExtractStream(formatRequired)) {
                 boolean isFirst = true;
                 Iterator<JsonObject> iterator = stream.iterator();
                 while (iterator.hasNext()) {
@@ -210,19 +214,21 @@ public ResponseEntity<StreamingResponseBody> streamJsonNative() {
                   isFirst = false;
                 }
               } catch (IOException e) {
-                LOG.error("Error during streaming jsonObjects using native mode: {}", e.getMessage());
+                LOG.error(
+                    "Error during streaming jsonObjects using native mode: {}", e.getMessage());
               }
             });
   }
 
   @GetMapping(value = "/inspections/asOf", produces = MediaType.APPLICATION_JSON_VALUE)
   public ResponseEntity<StreamingResponseBody> dataAtDate(
-          @RequestParam(name = "asOfDate") @DateTimeFormat(pattern = "yyyyMMddHHmmss") Date asOfDate,
-          @RequestParam(name = "id") Long id) {
+      @RequestParam(name = "asOfDate") @DateTimeFormat(pattern = "yyyyMMddHHmmss") Date asOfDate,
+      @RequestParam(name = "id") Long id) {
     return ResponseEntity.ok()
-            .contentType(MediaType.APPLICATION_JSON)
-            .body(outputStream ->
-                    writeDocumentsToOutputStream(outputStream, historyService.asOfDate(id, asOfDate)));
+        .contentType(MediaType.APPLICATION_JSON)
+        .body(
+            outputStream ->
+                writeDocumentsToOutputStream(outputStream, historyService.asOfDate(id, asOfDate)));
   }
 
   private void writeDocumentsToOutputStream(

memex/src/main/java/com/johnlpage/memex/controller/VehicleInspectionController.java

@@ -2,7 +2,7 @@
 
 import com.fasterxml.jackson.core.JsonFactory;
 import com.fasterxml.jackson.databind.ObjectMapper;
-import com.johnlpage.memex.model.UpdateStrategy;
+import com.johnlpage.memex.util.UpdateStrategy;
 import com.johnlpage.memex.model.VehicleInspection;
 import com.johnlpage.memex.repository.optimized.OptimizedMongoLoadRepository;
 import com.johnlpage.memex.service.VehicleInspectionInvalidDataHandlerService;

memex/src/main/java/com/johnlpage/memex/kafka/VehicleInspectionKafkaConsumer.java

@@ -1,6 +1,7 @@
 package com.johnlpage.memex.model;
 
 import com.fasterxml.jackson.annotation.*;
+import com.johnlpage.memex.util.DeleteFlag;
 import com.johnlpage.memex.util.ObjectConverter;
 import java.util.Date;
 import java.util.HashMap;
@@ -43,7 +44,7 @@ public class VehicleInspection {
   Vehicle vehicle;
   String files;
 
-  @Min(49)
+  @Min(1)
   Long capacity;
 
   Date firstusedate;

memex/src/main/java/com/johnlpage/memex/model/VehicleInspection.java

@@ -25,7 +25,7 @@ public interface VehicleInspectionRepository
    *
    *
   // Find inspections by engine capacity - auto generated query
-  List<VehicleInspection> findByCapacityGreaterThan(Long engineCapacity);
+  List<VehicleInspection> findAllByCapacityGreaterThan(Long engineCapacity);
 
   // Custom query to find vehicle inspections by vehicle make and model
   @Query("{ 'vehicle.make': ?0, 'vehicle.model': ?1 }")

memex/src/main/java/com/johnlpage/memex/repository/VehicleInspectionRepository.java

@@ -1,6 +1,6 @@
 package com.johnlpage.memex.repository.optimized;
 
-import com.johnlpage.memex.model.UpdateStrategy;
+import com.johnlpage.memex.util.UpdateStrategy;
 import com.johnlpage.memex.service.generic.InvalidDataHandlerService;
 import com.johnlpage.memex.service.generic.PostWriteTriggerService;
 import com.mongodb.bulk.BulkWriteResult;

memex/src/main/java/com/johnlpage/memex/repository/optimized/OptimizedMongoLoadRepository.java

@@ -3,7 +3,7 @@
 import static com.johnlpage.memex.util.AnnotationExtractor.*;
 import static org.springframework.data.mongodb.core.query.Criteria.where;
 
-import com.johnlpage.memex.model.UpdateStrategy;
+import com.johnlpage.memex.util.UpdateStrategy;
 import com.johnlpage.memex.service.generic.InvalidDataHandlerService;
 import com.johnlpage.memex.service.generic.PostWriteTriggerService;
 import com.mongodb.bulk.BulkWriteInsert;

memex/src/main/java/com/johnlpage/memex/repository/optimized/OptimizedMongoLoadRepositoryImpl.java

@@ -5,7 +5,7 @@
 import com.fasterxml.jackson.core.JsonToken;
 import com.fasterxml.jackson.databind.JsonNode;
 import com.fasterxml.jackson.databind.ObjectMapper;
-import com.johnlpage.memex.model.UpdateStrategy;
+import com.johnlpage.memex.util.UpdateStrategy;
 import com.johnlpage.memex.repository.optimized.OptimizedMongoLoadRepository;
 import com.mongodb.bulk.BulkWriteResult;
 import jakarta.annotation.Nullable;
@@ -24,17 +24,14 @@
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 import org.springframework.stereotype.Service;
-import jakarta.validation.ConstraintViolation;
-import jakarta.validation.Validation;
-import jakarta.validation.Validator;
-import jakarta.validation.ValidatorFactory;
 
 @Service
 @RequiredArgsConstructor
 public abstract class MongoDbJsonStreamingLoaderService<T> {
 
   private static final Logger LOG =
       LoggerFactory.getLogger(MongoDbJsonStreamingLoaderService.class);
+  private static final int BATCH_SIZE = 200;
   private final OptimizedMongoLoadRepository<T> repository;
   private final ObjectMapper objectMapper;
   private final JsonFactory jsonFactory;
@@ -79,7 +76,7 @@ public JsonStreamingLoadResponse loadFromJsonStream(
           count++;
 
           toSave.add(document);
-          if (toSave.size() >= 100) {
+          if (toSave.size() >= BATCH_SIZE) {
             List<T> copyOfToSave = List.copyOf(toSave);
             toSave.clear();
             futures.add(

memex/src/main/java/com/johnlpage/memex/service/generic/MongoDbJsonStreamingLoaderService.java

@@ -1,7 +1,6 @@
 package com.johnlpage.memex.util;
 
 import com.fasterxml.jackson.annotation.JsonProperty;
-import com.johnlpage.memex.model.DeleteFlag;
 import jakarta.annotation.Nullable;
 import java.lang.reflect.Field;
 import java.util.HashMap;

memex/src/main/java/com/johnlpage/memex/util/AnnotationExtractor.java

@@ -1,4 +1,4 @@
-package com.johnlpage.memex.model;
+package com.johnlpage.memex.util;
 
 import java.lang.annotation.ElementType;
 import java.lang.annotation.Retention;