Merge pull request #2 from gdcc/1-field

pdurbin · web-flow · commit 1687207f98b6 · 2024-06-03T14:19:16.000-04:00
stop repeating field over and over #1
diff --git a/README.md b/README.md
@@ -152,7 +152,6 @@ Same as above but use a JVM option in domain.xml such as the example below.
 ### Differences from Kaggle
 
 - I see an `encodingFormat` of `text/comma-separated-values`. Kind of curious about that since I think `text/csv` is more the MIME type that's on https://www.iana.org/assignments/media-types/media-types.xhtml and https://developer.mozilla.org/en-US/docs/Web/HTTP/Basics_of_HTTP/MIME_types . See https://github.com/IQSS/dataverse/issues/4943#issuecomment-2145333830
-- One big difference I see is that you have many `recordSets` (and each one containing a single `field`) despite there being only 1 CSV. My understanding was that a `recordSet` maps roughly to a table and a `field` maps roughly to a column. So you'll see that our implementation has only 1 `recordSet` with many `field`s. This might be a good thing to get clarification on.
 - Another thing that sticks out is that I see all of the `field`s have a `dataType` of `sc:Integer`. But nearly all of the columns (excluding `quality` and `Id`) are `sc:Float`. On the Kaggle side, we have a column type of "Id" and so if that's set on a column, we set the `dataType` to `sc:Text` since Ids can often be non-numerical. Just a minor difference there, though, so nothing alarming to me personally.
 
 ### Differences from pyDataverse
diff --git a/src/main/java/io/gdcc/spi/export/croissant/CroissantExporter.java b/src/main/java/io/gdcc/spi/export/croissant/CroissantExporter.java
@@ -193,6 +193,8 @@ public void exportDataset(ExportDataProvider dataProvider, OutputStream outputSt
             int fileCounter = 0;
             for (JsonValue jsonValue : datasetFileDetails) {
 
+                JsonObjectBuilder recordSetContent = Json.createObjectBuilder();
+                recordSetContent.add("@type", "cr:RecordSet");
                 JsonObject fileDetails = jsonValue.asJsonObject();
                 /**
                  * When there is an originalFileName, it means that the file has gone through ingest
@@ -306,9 +308,9 @@ public void exportDataset(ExportDataProvider dataProvider, OutputStream outputSt
                                                                 "fileObject",
                                                                 Json.createObjectBuilder()
                                                                         .add("@id", fileId))));
-                        fieldSetObject.add("field", fieldSetArray);
-                        recordSet.add(fieldSetObject);
                     }
+                    recordSetContent.add("field", fieldSetArray);
+                    recordSet.add(recordSetContent);
                     fileIndex++;
                 }
                 fileCounter++;
diff --git a/src/test/resources/cars/expected/cars-croissant.json b/src/test/resources/cars/expected/cars-croissant.json
@@ -126,12 +126,7 @@
                             "@id": "data/stata13-auto.dta"
                         }
                     }
-                }
-            ]
-        },
-        {
-            "@type": "cr:RecordSet",
-            "field": [
+                },
                 {
                     "@type": "cr:Field",
                     "name": "price",
@@ -143,12 +138,7 @@
                             "@id": "data/stata13-auto.dta"
                         }
                     }
-                }
-            ]
-        },
-        {
-            "@type": "cr:RecordSet",
-            "field": [
+                },
                 {
                     "@type": "cr:Field",
                     "name": "mpg",
@@ -160,12 +150,7 @@
                             "@id": "data/stata13-auto.dta"
                         }
                     }
-                }
-            ]
-        },
-        {
-            "@type": "cr:RecordSet",
-            "field": [
+                },
                 {
                     "@type": "cr:Field",
                     "name": "rep78",
@@ -177,12 +162,7 @@
                             "@id": "data/stata13-auto.dta"
                         }
                     }
-                }
-            ]
-        },
-        {
-            "@type": "cr:RecordSet",
-            "field": [
+                },
                 {
                     "@type": "cr:Field",
                     "name": "headroom",
@@ -194,12 +174,7 @@
                             "@id": "data/stata13-auto.dta"
                         }
                     }
-                }
-            ]
-        },
-        {
-            "@type": "cr:RecordSet",
-            "field": [
+                },
                 {
                     "@type": "cr:Field",
                     "name": "trunk",
@@ -211,12 +186,7 @@
                             "@id": "data/stata13-auto.dta"
                         }
                     }
-                }
-            ]
-        },
-        {
-            "@type": "cr:RecordSet",
-            "field": [
+                },
                 {
                     "@type": "cr:Field",
                     "name": "weight",
@@ -228,12 +198,7 @@
                             "@id": "data/stata13-auto.dta"
                         }
                     }
-                }
-            ]
-        },
-        {
-            "@type": "cr:RecordSet",
-            "field": [
+                },
                 {
                     "@type": "cr:Field",
                     "name": "length",
@@ -245,12 +210,7 @@
                             "@id": "data/stata13-auto.dta"
                         }
                     }
-                }
-            ]
-        },
-        {
-            "@type": "cr:RecordSet",
-            "field": [
+                },
                 {
                     "@type": "cr:Field",
                     "name": "turn",
@@ -262,12 +222,7 @@
                             "@id": "data/stata13-auto.dta"
                         }
                     }
-                }
-            ]
-        },
-        {
-            "@type": "cr:RecordSet",
-            "field": [
+                },
                 {
                     "@type": "cr:Field",
                     "name": "displacement",
@@ -279,12 +234,7 @@
                             "@id": "data/stata13-auto.dta"
                         }
                     }
-                }
-            ]
-        },
-        {
-            "@type": "cr:RecordSet",
-            "field": [
+                },
                 {
                     "@type": "cr:Field",
                     "name": "gear_ratio",
@@ -296,12 +246,7 @@
                             "@id": "data/stata13-auto.dta"
                         }
                     }
-                }
-            ]
-        },
-        {
-            "@type": "cr:RecordSet",
-            "field": [
+                },
                 {
                     "@type": "cr:Field",
                     "name": "foreign",

Original file line number	Diff line number	Diff line change
`@@ -126,12 +126,7 @@`
`126`	`126`	`"@id": "data/stata13-auto.dta"`
`127`	`127`	`}`
`128`	`128`	`}`
`129`		`- }`
`130`		`- ]`
`131`		`- },`
`132`		`- {`
`133`		`- "@type": "cr:RecordSet",`
`134`		`- "field": [`
	`129`	`+ },`
`135`	`130`	`{`
`136`	`131`	`"@type": "cr:Field",`
`137`	`132`	`"name": "price",`
`@@ -143,12 +138,7 @@`
`143`	`138`	`"@id": "data/stata13-auto.dta"`
`144`	`139`	`}`
`145`	`140`	`}`
`146`		`- }`
`147`		`- ]`
`148`		`- },`
`149`		`- {`
`150`		`- "@type": "cr:RecordSet",`
`151`		`- "field": [`
	`141`	`+ },`
`152`	`142`	`{`
`153`	`143`	`"@type": "cr:Field",`
`154`	`144`	`"name": "mpg",`
`@@ -160,12 +150,7 @@`
`160`	`150`	`"@id": "data/stata13-auto.dta"`
`161`	`151`	`}`
`162`	`152`	`}`
`163`		`- }`
`164`		`- ]`
`165`		`- },`
`166`		`- {`
`167`		`- "@type": "cr:RecordSet",`
`168`		`- "field": [`
	`153`	`+ },`
`169`	`154`	`{`
`170`	`155`	`"@type": "cr:Field",`
`171`	`156`	`"name": "rep78",`
`@@ -177,12 +162,7 @@`
`177`	`162`	`"@id": "data/stata13-auto.dta"`
`178`	`163`	`}`
`179`	`164`	`}`
`180`		`- }`
`181`		`- ]`
`182`		`- },`
`183`		`- {`
`184`		`- "@type": "cr:RecordSet",`
`185`		`- "field": [`
	`165`	`+ },`
`186`	`166`	`{`
`187`	`167`	`"@type": "cr:Field",`
`188`	`168`	`"name": "headroom",`
`@@ -194,12 +174,7 @@`
`194`	`174`	`"@id": "data/stata13-auto.dta"`
`195`	`175`	`}`
`196`	`176`	`}`
`197`		`- }`
`198`		`- ]`
`199`		`- },`
`200`		`- {`
`201`		`- "@type": "cr:RecordSet",`
`202`		`- "field": [`
	`177`	`+ },`
`203`	`178`	`{`
`204`	`179`	`"@type": "cr:Field",`
`205`	`180`	`"name": "trunk",`
`@@ -211,12 +186,7 @@`
`211`	`186`	`"@id": "data/stata13-auto.dta"`
`212`	`187`	`}`
`213`	`188`	`}`
`214`		`- }`
`215`		`- ]`
`216`		`- },`
`217`		`- {`
`218`		`- "@type": "cr:RecordSet",`
`219`		`- "field": [`
	`189`	`+ },`
`220`	`190`	`{`
`221`	`191`	`"@type": "cr:Field",`
`222`	`192`	`"name": "weight",`
`@@ -228,12 +198,7 @@`
`228`	`198`	`"@id": "data/stata13-auto.dta"`
`229`	`199`	`}`
`230`	`200`	`}`
`231`		`- }`
`232`		`- ]`
`233`		`- },`
`234`		`- {`
`235`		`- "@type": "cr:RecordSet",`
`236`		`- "field": [`
	`201`	`+ },`
`237`	`202`	`{`
`238`	`203`	`"@type": "cr:Field",`
`239`	`204`	`"name": "length",`
`@@ -245,12 +210,7 @@`
`245`	`210`	`"@id": "data/stata13-auto.dta"`
`246`	`211`	`}`
`247`	`212`	`}`
`248`		`- }`
`249`		`- ]`
`250`		`- },`
`251`		`- {`
`252`		`- "@type": "cr:RecordSet",`
`253`		`- "field": [`
	`213`	`+ },`
`254`	`214`	`{`
`255`	`215`	`"@type": "cr:Field",`
`256`	`216`	`"name": "turn",`
`@@ -262,12 +222,7 @@`
`262`	`222`	`"@id": "data/stata13-auto.dta"`
`263`	`223`	`}`
`264`	`224`	`}`
`265`		`- }`
`266`		`- ]`
`267`		`- },`
`268`		`- {`
`269`		`- "@type": "cr:RecordSet",`
`270`		`- "field": [`
	`225`	`+ },`
`271`	`226`	`{`
`272`	`227`	`"@type": "cr:Field",`
`273`	`228`	`"name": "displacement",`
`@@ -279,12 +234,7 @@`
`279`	`234`	`"@id": "data/stata13-auto.dta"`
`280`	`235`	`}`
`281`	`236`	`}`
`282`		`- }`
`283`		`- ]`
`284`		`- },`
`285`		`- {`
`286`		`- "@type": "cr:RecordSet",`
`287`		`- "field": [`
	`237`	`+ },`
`288`	`238`	`{`
`289`	`239`	`"@type": "cr:Field",`
`290`	`240`	`"name": "gear_ratio",`
`@@ -296,12 +246,7 @@`
`296`	`246`	`"@id": "data/stata13-auto.dta"`
`297`	`247`	`}`
`298`	`248`	`}`
`299`		`- }`
`300`		`- ]`
`301`		`- },`
`302`		`- {`
`303`		`- "@type": "cr:RecordSet",`
`304`		`- "field": [`
	`249`	`+ },`
`305`	`250`	`{`
`306`	`251`	`"@type": "cr:Field",`
`307`	`252`	`"name": "foreign",`