Add paging to dataset versions panel (#2855)

* initial commit of dataset paging

Signed-off-by: sharpd <davidsharp7@gmail.com>

* fix linting

Signed-off-by: sharpd <davidsharp7@gmail.com>

* remove system.out

Signed-off-by: sharpd <davidsharp7@gmail.com>

* add dataset versions count test

Signed-off-by: sharpd <davidsharp7@gmail.com>

* remove system print

Signed-off-by: sharpd <davidsharp7@gmail.com>

---------

Signed-off-by: sharpd <davidsharp7@gmail.com>

Author: davidsharp7

api/src/main/java/marquez/api/DatasetResource.java

@@ -133,7 +133,11 @@ public Response listVersions(
     final List<DatasetVersion> datasetVersions =
         datasetVersionService.findAllWithRun(
             namespaceName.getValue(), datasetName.getValue(), limit, offset);
-    return Response.ok(new DatasetVersions(datasetVersions)).build();
+
+    final int totalCount =
+        datasetVersionService.countDatasetVersions(
+            namespaceName.getValue(), datasetName.getValue());
+    return Response.ok(new DatasetVersions(datasetVersions, totalCount)).build();
   }
 
   @Timed
@@ -301,5 +305,8 @@ static class DatasetVersions {
     @NonNull
     @JsonProperty("versions")
     List<DatasetVersion> value;
+
+    @JsonProperty("totalCount")
+    int totalCount;
   }
 }

api/src/main/java/marquez/db/DatasetVersionDao.java

@@ -315,6 +315,20 @@ default List<DatasetVersion> findAllWithRun(
   @SqlQuery(SELECT + "WHERE dv.uuid = :uuid")
   Optional<DatasetVersionRow> findRowByUuid(UUID uuid);
 
+  @SqlQuery(
+      """
+    select
+        count(*)
+    from
+        dataset_versions
+    where
+        namespace_name = :namespaceName
+    and
+        dataset_name = :dataset
+    ;
+    """)
+  int countDatasetVersions(String namespaceName, String dataset);
+
   @SqlQuery(
       "INSERT INTO dataset_versions "
           + "(uuid, created_at, dataset_uuid, version, dataset_schema_version_uuid, run_uuid, fields, namespace_name, dataset_name, lifecycle_state) "

api/src/test/java/marquez/db/DatasetDaoTest.java

@@ -114,6 +114,10 @@ void testUpsertMetadataMultipleRuns() {
     assertEquals(firstResult.getId(), secondResult.getId());
     assertNotEquals(firstResult.getCurrentVersion(), secondResult.getCurrentVersion());
     assertEquals(resolveSchemaVersion(firstResult), resolveSchemaVersion(secondResult));
+
+    // test count of dataset versions
+    int jobCount = datasetVersionDao.countDatasetVersions(NAMESPACE, DATASET);
+    assertEquals(jobCount, 2);
   }
 
   @Test
@@ -310,6 +314,7 @@ public void testGetDatasetWithMultipleVersions() {
 
     Optional<marquez.service.models.Dataset> datasetByName =
         datasetDao.findDatasetByName(NAMESPACE, DATASET);
+
     assertThat(datasetByName)
         .isPresent()
         .get()

web/src/components/datasets/DatasetDetailPage.tsx

@@ -28,7 +28,7 @@ import {
   deleteDataset,
   dialogToggle,
   fetchDataset,
-  fetchDatasetVersions,
+  fetchInitialDatasetVersions,
   resetDataset,
   resetDatasetVersions,
   setTabIndex,
@@ -56,15 +56,15 @@ import StorageIcon from '@mui/icons-material/Storage'
 interface StateProps {
   lineageDataset: LineageDataset
   dataset: Dataset
-  versions: DatasetVersion[]
-  versionsLoading: boolean
+  initVersions: DatasetVersion[]
+  initVersionsLoading: boolean
   datasets: IState['datasets']
   display: IState['display']
   tabIndex: IState['lineage']['tabIndex']
 }
 
 interface DispatchProps {
-  fetchDatasetVersions: typeof fetchDatasetVersions
+  fetchInitialDatasetVersions: typeof fetchInitialDatasetVersions
   fetchDataset: typeof fetchDataset
   resetDatasetVersions: typeof resetDatasetVersions
   resetDataset: typeof resetDataset
@@ -88,13 +88,13 @@ const DatasetDetailPage: FunctionComponent<IProps> = (props) => {
     dataset,
     display,
     fetchDataset,
-    fetchDatasetVersions,
     resetDataset,
     resetDatasetVersions,
+    fetchInitialDatasetVersions,
     deleteDataset,
     dialogToggle,
-    versions,
-    versionsLoading,
+    initVersions,
+    initVersionsLoading,
     lineageDataset,
     tabIndex,
     setTabIndex,
@@ -114,8 +114,9 @@ const DatasetDetailPage: FunctionComponent<IProps> = (props) => {
     []
   )
 
+  // might need to map first version to its own state
   useEffect(() => {
-    fetchDatasetVersions(lineageDataset.namespace, lineageDataset.name)
+    fetchInitialDatasetVersions(lineageDataset.namespace, lineageDataset.name)
     fetchDataset(lineageDataset.namespace, lineageDataset.name)
   }, [lineageDataset.name, showTags])
 
@@ -130,19 +131,19 @@ const DatasetDetailPage: FunctionComponent<IProps> = (props) => {
     setTabIndex(newValue)
   }
 
-  if (versionsLoading && versions.length === 0) {
+  if (initVersionsLoading && initVersions.length === 0) {
     return (
       <Box display={'flex'} justifyContent={'center'} mt={2}>
         <CircularProgress color='primary' />
       </Box>
     )
   }
 
-  if (versions.length === 0) {
+  if (initVersions.length === 0) {
     return null
   }
 
-  const firstVersion = versions[0]
+  const firstVersion = initVersions[0]
   const { name, tags, description } = firstVersion
   const facetsStatus = datasetFacetsStatus(firstVersion.facets)
 
@@ -326,7 +327,7 @@ const DatasetDetailPage: FunctionComponent<IProps> = (props) => {
                   checked={showTags}
                   onChange={() => setShowTags(!showTags)}
                   inputProps={{ 'aria-label': 'toggle show tags' }}
-                  disabled={versionsLoading}
+                  disabled={initVersionsLoading}
                 />
               }
               label={i18next.t('datasets.show_field_tags')}
@@ -344,7 +345,7 @@ const DatasetDetailPage: FunctionComponent<IProps> = (props) => {
           isCurrentVersion
         />
       )}
-      {tabIndex === 1 && <DatasetVersions dataset={dataset} versions={props.versions} />}
+      {tabIndex === 1 && <DatasetVersions dataset={dataset} />}
     </Box>
   )
 }
@@ -353,15 +354,15 @@ const mapStateToProps = (state: IState) => ({
   datasets: state.datasets,
   dataset: state.dataset.result,
   display: state.display,
-  versions: state.datasetVersions.result.versions,
-  versionsLoading: state.datasetVersions.isLoading,
+  initVersions: state.datasetVersions.initDsVersion.versions,
+  initVersionsLoading: state.datasetVersions.isInitDsVerLoading,
   tabIndex: state.lineage.tabIndex,
 })
 
 const mapDispatchToProps = (dispatch: Redux.Dispatch) =>
   bindActionCreators(
     {
-      fetchDatasetVersions: fetchDatasetVersions,
+      fetchInitialDatasetVersions: fetchInitialDatasetVersions,
       fetchDataset: fetchDataset,
       resetDatasetVersions: resetDatasetVersions,
       resetDataset: resetDataset,

web/src/components/datasets/DatasetInfo.tsx

@@ -44,9 +44,11 @@ const DatasetInfo: FunctionComponent<DatasetInfoProps> = (props) => {
   const { datasetFields, facets, run, dataset, fetchJobFacets, resetFacets, showTags } = props
   const i18next = require('i18next')
   const dsNamespace = useSelector(
-    (state: IState) => state.datasetVersions.result.versions[0].namespace
+    (state: IState) => state.datasetVersions.initDsVersion.versions[0].namespace
+  )
+  const dsName = useSelector(
+    (state: IState) => state.datasetVersions.initDsVersion.versions[0].name
   )
-  const dsName = useSelector((state: IState) => state.datasetVersions.result.versions[0].name)
 
   useEffect(() => {
     run && fetchJobFacets(run.id)