display column lineage JSON

Signed-off-by: Pawel Leszczynski <leszczynski.pawel@gmail.com>

Author: pawel-big-lebowski

web/src/components/bottom-bar/BottomBar.tsx

@@ -54,7 +54,7 @@ class BottomBar extends React.Component<BottomBarProps> {
         <Box className={classes.overflow} height={bottomBarHeight}>
           <Container maxWidth={'lg'} disableGutters={true}>
             {lineageJob && <JobDetailPage job={lineageJob} />}
-            {lineageDataset && <DatasetDetailPage dataset={lineageDataset} />}
+            {lineageDataset && <DatasetDetailPage lineageDataset={lineageDataset} />}
           </Container>
         </Box>
       </Box>

web/src/components/datasets/DatasetColumnLineage.tsx

@@ -0,0 +1,27 @@
+// SPDX-License-Identifier: Apache-2.0
+
+import { Box } from '@material-ui/core'
+import MqEmpty from '../core/empty/MqEmpty'
+import MqJson from '../core/code/MqJson'
+import React, { FunctionComponent } from 'react'
+
+interface DatasetColumnLineageProps {
+  columnLineage: object
+}
+
+const DatasetColumnLineage: FunctionComponent<DatasetColumnLineageProps> = props => {
+  const { columnLineage } = props
+
+  return (
+    <Box>
+      {columnLineage === null && <MqEmpty title={'No column lineage'} body={'Column lineage not available for the specified dataset.'} />}
+      {columnLineage && (
+        <Box mt={2}>
+          <MqJson code={columnLineage} />
+        </Box>
+      )}
+    </Box>
+  )
+}
+
+export default DatasetColumnLineage

web/src/components/datasets/DatasetDetailPage.tsx

@@ -4,7 +4,7 @@ import React, { ChangeEvent, FunctionComponent, SetStateAction, useEffect } from
 
 import * as Redux from 'redux'
 import { Box, Chip, Tab, Tabs } from '@material-ui/core'
-import { DatasetVersion } from '../../types/api'
+import { Dataset, DatasetVersion } from '../../types/api';
 import { IState } from '../../store/reducers'
 import {
   Theme as ITheme,
@@ -15,10 +15,15 @@ import {
 import { LineageDataset } from '../lineage/types'
 import { bindActionCreators } from 'redux'
 import { connect } from 'react-redux'
-import { fetchDatasetVersions, resetDatasetVersions } from '../../store/actionCreators'
+import {
+  fetchDataset,
+  fetchDatasetVersions,
+  resetDatasetVersions
+} from '../../store/actionCreators'
 import { useHistory } from 'react-router-dom'
 import CircularProgress from '@material-ui/core/CircularProgress/CircularProgress'
 import CloseIcon from '@material-ui/icons/Close'
+import DatasetColumnLineage from './DatasetColumnLineage'
 import DatasetInfo from './DatasetInfo'
 import DatasetVersions from './DatasetVersions'
 import IconButton from '@material-ui/core/IconButton'
@@ -55,13 +60,15 @@ const styles = ({ spacing }: ITheme) => {
 }
 
 interface StateProps {
-  dataset: LineageDataset
+  lineageDataset: LineageDataset
   versions: DatasetVersion[]
+  dataset: Dataset
   versionsLoading: boolean
 }
 
 interface DispatchProps {
   fetchDatasetVersions: typeof fetchDatasetVersions
+  fetchDataset: typeof fetchDataset
   resetDatasetVersions: typeof resetDatasetVersions
 }
 
@@ -75,13 +82,13 @@ function a11yProps(index: number) {
 }
 
 const DatasetDetailPage: FunctionComponent<IProps> = props => {
-  const { classes, fetchDatasetVersions, resetDatasetVersions, versions, versionsLoading } = props
+  const { classes, fetchDatasetVersions, fetchDataset, resetDatasetVersions, versions, versionsLoading } = props
   const { root } = classes
   const history = useHistory()
 
   useEffect(() => {
-    fetchDatasetVersions(props.dataset.namespace, props.dataset.name)
-  }, [props.dataset.name])
+    fetchDatasetVersions(props.lineageDataset.namespace, props.lineageDataset.name)
+  }, [props.lineageDataset.name])
 
   // unmounting
   useEffect(() => {
@@ -90,6 +97,10 @@ const DatasetDetailPage: FunctionComponent<IProps> = props => {
     }
   }, [])
 
+  // useEffect(() => {
+  //   fetchDataset(props.lineageDataset.namespace, props.lineageDataset.name)
+  // }, [props.lineageDataset.name])
+
   const [tab, setTab] = React.useState(0)
   const handleChange = (event: ChangeEvent, newValue: SetStateAction<number>) => {
     setTab(newValue)
@@ -107,8 +118,8 @@ const DatasetDetailPage: FunctionComponent<IProps> = props => {
     return null
   }
 
-  const dataset = versions[0]
-  const { name, tags, description } = dataset
+  const firstVersion = versions[0]
+  const { name, tags, description } = firstVersion
 
   return (
     <Box my={2} className={root}>
@@ -127,6 +138,7 @@ const DatasetDetailPage: FunctionComponent<IProps> = props => {
             <Tabs value={tab} onChange={handleChange} textColor='primary' indicatorColor='primary'>
               <Tab label='LATEST SCHEMA' {...a11yProps(0)} disableRipple={true} />
               <Tab label='VERSION HISTORY' {...a11yProps(1)} disableRipple={true} />
+              <Tab label='COLUMN LINEAGE' {...a11yProps(2)} disableRipple={true} />
             </Tabs>
           </Box>
           <IconButton onClick={() => history.push('/datasets')}>
@@ -142,26 +154,29 @@ const DatasetDetailPage: FunctionComponent<IProps> = props => {
       </Box>
       {tab === 0 && (
         <DatasetInfo
-          datasetFields={dataset.fields}
-          facets={dataset.facets}
-          run={dataset.createdByRun}
+          datasetFields={firstVersion.fields}
+          facets={firstVersion.facets}
+          run={firstVersion.createdByRun}
         />
       )}
       {tab === 1 && <DatasetVersions versions={props.versions} />}
+      {tab === 2 && (props.dataset.name !== undefined || fetchDataset(props.lineageDataset.namespace, props.lineageDataset.name)) && <DatasetColumnLineage columnLineage={props.dataset.columnLineage} />}
     </Box>
   )
 }
 
 const mapStateToProps = (state: IState) => ({
   datasets: state.datasets.result,
   versions: state.datasetVersions.result.versions,
+  dataset: state.dataset.result,
   versionsLoading: state.datasetVersions.isLoading
 })
 
 const mapDispatchToProps = (dispatch: Redux.Dispatch) =>
   bindActionCreators(
     {
       fetchDatasetVersions: fetchDatasetVersions,
+      fetchDataset: fetchDataset,
       resetDatasetVersions: resetDatasetVersions
     },
     dispatch

web/src/store/actionCreators/actionTypes.ts

@@ -22,8 +22,11 @@ export const RESET_JOBS = 'RESET_JOBS'
 
 // datasets
 export const FETCH_DATASETS = 'FETCH_DATASETS'
+export const FETCH_DATASET = 'FETCH_DATASET'
 export const FETCH_DATASETS_SUCCESS = 'FETCH_DATASETS_SUCCESS'
+export const FETCH_DATASET_SUCCESS = 'FETCH_DATASET_SUCCESS'
 export const RESET_DATASETS = 'RESET_DATASETS'
+export const RESET_DATASET = 'RESET_DATASET'
 export const FETCH_DATASET_VERSIONS = 'FETCH_DATASET_VERSIONS'
 export const FETCH_DATASET_VERSIONS_SUCCESS = 'FETCH_DATASET_VERSIONS_SUCCESS'
 export const RESET_DATASET_VERSIONS = 'RESET_DATASET_VERSIONS'

web/src/store/actionCreators/index.ts

@@ -39,6 +39,21 @@ export const fetchDatasetsSuccess = (datasets: Dataset[]) => ({
   }
 })
 
+export const fetchDataset = (namespace: string, name: string) => ({
+  type: actionTypes.FETCH_DATASET,
+  payload: {
+    namespace,
+    name
+  }
+})
+
+export const fetchDatasetSuccess = (dataset: Dataset) => ({
+  type: actionTypes.FETCH_DATASET_SUCCESS,
+  payload: {
+    dataset
+  }
+})
+
 export const fetchDatasetVersions = (namespace: string, name: string) => ({
   type: actionTypes.FETCH_DATASET_VERSIONS,
   payload: {

web/src/store/reducers/dataset.ts

@@ -0,0 +1,30 @@
+// SPDX-License-Identifier: Apache-2.0
+
+import { Dataset } from '../../types/api'
+import {
+  FETCH_DATASET,
+  FETCH_DATASET_SUCCESS,
+  RESET_DATASET
+} from '../actionCreators/actionTypes'
+import { fetchDatasetSuccess } from '../actionCreators';
+
+export type IDatasetState = { isLoading: boolean; result: Dataset; init: boolean }
+
+export const initialState: IDatasetState = { isLoading: false, init: false, result: { } as Dataset }
+
+type IDatasetAction = ReturnType<typeof fetchDatasetSuccess>
+
+export default (state: IDatasetState = initialState, action: IDatasetAction): IDatasetState => {
+  const { type, payload } = action
+
+  switch (type) {
+    case FETCH_DATASET:
+      return { ...state, isLoading: true }
+    case FETCH_DATASET_SUCCESS:
+      return { ...state, isLoading: false, init: true, result: payload.dataset }
+    case RESET_DATASET:
+      return initialState
+    default:
+      return state
+  }
+}

web/src/store/reducers/index.ts

@@ -3,6 +3,7 @@
 import { History } from 'history'
 import { Reducer, combineReducers } from 'redux'
 import { connectRouter } from 'connected-react-router'
+import dataset, { IDatasetState } from './dataset';
 import datasetVersions, { IDatasetVersionsState } from './datasetVersions'
 import datasets, { IDatasetsState } from './datasets'
 import events, { IEventsState } from './events'
@@ -15,6 +16,7 @@ import search, { ISearchState } from './search'
 
 export interface IState {
   datasets: IDatasetsState
+  dataset: IDatasetState
   datasetVersions: IDatasetVersionsState
   events: IEventsState
   jobs: IJobsState
@@ -29,6 +31,7 @@ export interface IState {
 export default (history: History): Reducer =>
   combineReducers({
     router: connectRouter(history),
+    dataset,
     datasets,
     datasetVersions,
     events,

web/src/store/requests/datasets.ts

@@ -1,7 +1,7 @@
 // SPDX-License-Identifier: Apache-2.0
 
 import { API_URL } from '../../globals'
-import { DatasetVersions, Datasets } from '../../types/api'
+import { Dataset, DatasetVersions, Datasets } from "../../types/api";
 import { genericFetchWrapper } from './index'
 
 export const getDatasets = async (namespace: string, limit = 25, offset = 0) => {
@@ -26,3 +26,10 @@ export const getDatasetVersions = async (
     (versions: DatasetVersions) => versions.versions
   )
 }
+
+export const getDataset = async (namespace: string, dataset: string) => {
+  const url = `${API_URL}/namespaces/${encodeURIComponent(namespace)}/datasets/${encodeURIComponent(
+    dataset
+  )}`
+  return genericFetchWrapper(url, { method: 'GET' }, 'fetchDataset').then((d: Dataset) => d)
+}

web/src/store/sagas/index.ts

@@ -8,8 +8,8 @@ import {
   FETCH_LINEAGE,
   FETCH_RUNS,
   FETCH_SEARCH,
-  FETCH_EVENTS
-} from '../actionCreators/actionTypes'
+  FETCH_EVENTS, FETCH_DATASET
+} from "../actionCreators/actionTypes";
 import { Namespaces } from '../../types/api'
 import { all, put, take } from 'redux-saga/effects'
 
@@ -24,9 +24,9 @@ import {
   fetchLineageSuccess,
   fetchNamespacesSuccess,
   fetchRunsSuccess,
-  fetchSearchSuccess
-} from '../actionCreators'
-import { getDatasetVersions, getDatasets, getEvents, getJobs, getNamespaces, getRuns } from '../requests'
+  fetchSearchSuccess, fetchDatasetSuccess
+} from "../actionCreators";
+import { getDatasetVersions, getDatasets, getDataset, getEvents, getJobs, getNamespaces, getRuns } from '../requests'
 import { getLineage } from '../requests/lineage'
 import { getSearch } from '../requests/search'
 
@@ -112,6 +112,18 @@ export function* fetchEventsSaga() {
   }
 }
 
+export function* fetchDatasetSaga() {
+  while (true) {
+    try {
+      const { payload } = yield take(FETCH_DATASET)
+      const datasets = yield call(getDataset, payload.namespace, payload.name)
+      yield put(fetchDatasetSuccess(datasets))
+    } catch (e) {
+      yield put(applicationError('Something went wrong while fetching dataset'))
+    }
+  }
+}
+
 export function* fetchDatasetVersionsSaga() {
   while (true) {
     try {
@@ -130,6 +142,7 @@ export default function* rootSaga(): Generator {
     fetchJobsSaga(),
     fetchRunsSaga(),
     fetchDatasetsSaga(),
+    fetchDatasetSaga(),
     fetchDatasetVersionsSaga(),
     fetchEventsSaga(),
     fetchLineage(),

web/src/types/api.ts

@@ -72,6 +72,7 @@ export interface Dataset {
   description: string
   facets: object
   deleted: boolean
+  columnLineage: object
 }
 
 export interface DatasetVersions {