Arrays, although not unwound in history now save only changed elements putting a placeholder of MinKey in for unchanged values - works when saving and fetching - this means a top level array that isn't modified much won't explode the history - nested arrays still might, but that's a whole other level of challenge.

Author: johnlpage

memex/src/main/java/com/johnlpage/memex/generics/repository/MongoHistoryRepositoryImpl.java

@@ -333,8 +333,61 @@ private void useSmartUpdate(
                 // If changed record the old value otherwise record nothing
                 Document coerceEmptyToNull =
                         new Document("$ifNull", Arrays.asList("$" + entry.getKey(), null));
+
+                /*EXPERIMENTAL */
+                // Check if the NEW value being set is an array - if so, do element-wise diff
+
+                Document previousValueExpr;
+                if (entry.getValue() instanceof List) {
+                    // Build an element-wise comparison:
+                    // For each index in the OLD array, if the element equals the corresponding
+                    // new element, store MinKey; otherwise store the old element.
+                    // Also handles the case where array lengths differ.
+                    //
+                    // We use $map over the old array with index, comparing each element
+                    // to the new array at the same index.
+
+                    // $range(0, $size(oldArray))
+                    Document oldArrayRef = new Document("$ifNull",
+                            Arrays.asList("$" + entry.getKey(), new ArrayList<>()));
+                    Document oldSize = new Document("$size", oldArrayRef);
+
+                    // The new value as a literal (in case it contains $ strings)
+                    Object newValueLiteral = entry.getValue();
+
+                    // $map over indices of the old array
+                    // For each index i:
+                    //   if oldArray[i] == newArray[i] -> MinKey
+                    //   else -> oldArray[i]
+                    Document oldElem = new Document("$arrayElemAt",
+                            Arrays.asList(oldArrayRef, "$$idx"));
+                    Document newElem = new Document("$arrayElemAt",
+                            Arrays.asList(new Document("$literal", newValueLiteral), "$$idx"));
+
+                    Document elemEqual = new Document("$eq", Arrays.asList(oldElem, newElem));
+                    // MinKey as a constant - use $literal with a MinKey BSON value
+                    org.bson.types.MinKey minKey = new org.bson.types.MinKey();
+                    Document conditionalElem = new Document("$cond",
+                            Arrays.asList(elemEqual, minKey, oldElem));
+
+                    Document mappedArray = new Document("$map", new Document()
+                            .append("input", new Document("$range",
+                                    Arrays.asList(0, oldSize)))
+                            .append("as", "idx")
+                            .append("in", conditionalElem));
+
+                    previousValueExpr = mappedArray;
+                } else {
+                    previousValueExpr = coerceEmptyToNull;
+                }
+                /*EXPERIMENTAL */
+
+                /* WAS
+                Document conditionalOnChange =
+                        new Document("$cond", Arrays.asList(valueChanged, coerceEmptyToNull, "$$REMOVE"));*/
+
                 Document conditionalOnChange =
-                        new Document("$cond", Arrays.asList(valueChanged, coerceEmptyToNull, "$$REMOVE"));
+                        new Document("$cond", Arrays.asList(valueChanged, previousValueExpr, "$$REMOVE"));
 
                 previousValues.append(PREVIOUS_VALS + "." + entry.getKey(), conditionalOnChange);
             }

memex/src/main/java/com/johnlpage/memex/generics/repository/OptimizedMongoLoadRepositoryImpl.java

@@ -89,6 +89,17 @@ public void responseShouldContain(String expectedSubstring) {
         response.then().body(containsString(expectedSubstring));
     }
 
+    @Then("the response should contain {string}: [{string}, {string}, {string}]")
+    public void the_response_should_contain(String field, String val1, String val2, String val3) {
+        List<String> expectedValues = List.of(val1, val2, val3);
+
+        // Extract actual values from your response
+        List<String> actualValues = response.jsonPath().getList(field);
+
+        assertEquals(expectedValues, actualValues);
+    }
+
+
     @Then("the response should contain {string} with {int} items")
     public void responseShouldContainContentWithItems(String key, int expectedCount) {
         assertNotNull(response.getBody(), "Response body should not be null");

memex/src/test/java/com/johnlpage/memex/cucumber/steps/RestApiSteps.java

@@ -7,11 +7,11 @@ Feature: Vehicle Inspection REST API - Point-in-Time History (As Of)
   Scenario: Successfully retrieve vehicle inspection history as of a specific date
     Given the following vehicle inspections exist:
       | vehicleinspection                                                |
-      | {"testid": 10001, "vehicle": {"make": "Ford", "model": "Focus"}} |
+      | {"testid": 10001, "vehicle": {"make": "Ford", "model": "Focus"},"faileditems": ["Handbrake", "Lights","Wipers"]} |
     And I wait for 1 second
     And I capture the current timestamp to "<timestamp>" with "yyyyMMddHHmmss" pattern
     And I wait for 1 second
-    And I send a POST request to "/api/inspections?updateStrategy=UPDATEWITHHISTORY&futz=true" with the payload:
+    And I send a POST request to "/api/inspections?updateStrategy=UPDATEWITHHISTORY" with the payload:
       """
       [
         {
@@ -23,7 +23,7 @@ Feature: Vehicle Inspection REST API - Point-in-Time History (As Of)
           "testmileage": 60000,
           "postcode": "SW1A 0AB",
           "fuel": "Diesel",
-          "capacity": 80,
+          "capacity": 90,
           "firstusedate": "2019-03-20T00:00:00Z",
           "faileditems": ["Brakes", "Lights"],
           "vehicle": {
@@ -41,4 +41,6 @@ Feature: Vehicle Inspection REST API - Point-in-Time History (As Of)
     And the "Transfer-Encoding" header should be "chunked"
     And the "Content-Type" header should be "application/json"
     And the response should contain "testid": 10001
-    And the response should contain "combined.vehicle.model": "Focus"
+    And the response should contain "vehicle.model": "Focus"
+    And the response should contain "faileditems": ["Handbrake", "Lights", "Wipers"]
+

memex/src/test/resources/features/inspections.rest.asof.feature

@@ -54,17 +54,10 @@ public class __className__Controller {
     private final ObjectMapper objectMapper;
     private final __className__Repository repository;
 
-    /**
-     * Create or update a single record
-     */
-    @PostMapping("/__singularApiPath__")
-    public void createOne(@RequestBody __className__ record) {
-        LOG.info("Saving __className__: {}", record);
-        repository.save(record);
-    }
+
 
     /**
-     * Bulk load from JSON stream.
+     * Bulk load from JSON stream. Could take a single item too
      * This could read from a file, Kafka queue, or any stream of JSON data.
      */
     @PostMapping("/__apiPath__")

memex/templates/controller/Controller.java.template

@@ -228,7 +228,6 @@ println "Generated: ${generatedCount}, Skipped: ${skippedCount}"
 println "========================================="
 println ""
 println "API Endpoints created:"
-println "  POST   /api/${singularApiPath}      - Create single record"
 println "  POST   /api/${apiPath}              - Bulk load from JSON stream"
 println "  GET    /api/${apiPath}/id/{id}      - Get by ID"
 println "  GET    /api/${apiPath}/byExample    - Example query with pagination"