Skip to content

Commit 39bcd3a

Browse files
pawel-big-lebowskitito12tito1212Pawel Leszczynski
authored
present column lineage of a dataset (#2293)
* display column lineage JSON Signed-off-by: Pawel Leszczynski <leszczynski.pawel@gmail.com> * refactoring for display column lineage JSON (#2294) Signed-off-by: tito1212 <vladyslav.sedenko@getindata.com> Signed-off-by: tito1212 <vladyslav.sedenko@getindata.com> Co-authored-by: tito1212 <vladyslav.sedenko@getindata.com> * Refactoring for display column lineage JSON (#2301) * refactoring for display column lineage JSON Signed-off-by: tito1212 <vladyslav.sedenko@getindata.com> * request api after click Signed-off-by: tito1212 <vladyslav.sedenko@getindata.com> Signed-off-by: tito1212 <vladyslav.sedenko@getindata.com> Co-authored-by: tito1212 <vladyslav.sedenko@getindata.com> * call backend on tab switch Signed-off-by: Pawel Leszczynski <leszczynski.pawel@gmail.com> * changes after review. refactoring. fix type and name of file Signed-off-by: tito12 <vladyslav.sedenko@gmail.com> * Changes after review. Separate fetching data in child component Signed-off-by: tito12 <vladyslav.sedenko@gmail.com> * eslint fix changes Signed-off-by: tito12 <vladyslav.sedenko@gmail.com> Signed-off-by: Pawel Leszczynski <leszczynski.pawel@gmail.com> Signed-off-by: tito1212 <vladyslav.sedenko@getindata.com> Signed-off-by: tito12 <vladyslav.sedenko@gmail.com> Co-authored-by: tito12 <vladyslav.sedenko@gmail.com> Co-authored-by: tito1212 <vladyslav.sedenko@getindata.com> Co-authored-by: Pawel Leszczynski <pawelleszczynski@Pawels-MacBook-Pro.local>
1 parent 65669ca commit 39bcd3a

13 files changed

Lines changed: 230 additions & 33 deletions

File tree

‎CHANGELOG.md‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,11 @@
66

77
### Added
88

9+
* Column-lineage endpoints supports point-in-time requests [`#2265`](https://github.com/MarquezProject/marquez/pull/2265) [@pawel-big-lebowski](https://github.com/pawel-big-lebowski)
10+
*Enable requesting `column-lineage` endpoint by a dataset version, job version or dataset field of a specific dataset version.*
11+
* Present column lineage of a dataset [`#2293`](https://github.com/MarquezProject/marquez/pull/2293) [@pawel-big-lebowski](https://github.com/pawel-big-lebowski)
12+
*Column lineage of a dataset with a single level of depth can
13+
be displayed in datase details tab.*
914
* Add point-in-time requests support to column-lineage endpoints [`#2265`](https://github.com/MarquezProject/marquez/pull/2265) [@pawel-big-lebowski](https://github.com/pawel-big-lebowski)
1015
*Enables requesting `column-lineage` endpoint by a dataset version, job version or dataset field of a specific dataset version.*
1116
* Add column lineage point-in-time Java client methods [`#2269`](https://github.com/MarquezProject/marquez/pull/2269) [@pawel-big-lebowski](https://github.com/pawel-big-lebowski)

‎web/src/components/bottom-bar/BottomBar.tsx‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -54,7 +54,7 @@ class BottomBar extends React.Component<BottomBarProps> {
5454
<Box className={classes.overflow} height={bottomBarHeight}>
5555
<Container maxWidth={'lg'} disableGutters={true}>
5656
{lineageJob && <JobDetailPage job={lineageJob} />}
57-
{lineageDataset && <DatasetDetailPage dataset={lineageDataset} />}
57+
{lineageDataset && <DatasetDetailPage lineageDataset={lineageDataset} />}
5858
</Container>
5959
</Box>
6060
</Box>

‎web/src/components/datasets/DatasetColumnLineage.tsx‎

Lines changed: 105 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,105 @@
1+
// SPDX-License-Identifier: Apache-2.0
2+
3+
import * as Redux from 'redux'
4+
import { Box, Button } from '@material-ui/core'
5+
import { Dataset } from '../../types/api'
6+
import { IState } from '../../store/reducers'
7+
import { LineageDataset } from '../lineage/types'
8+
import { bindActionCreators } from 'redux'
9+
import { connect } from 'react-redux'
10+
import { fetchDataset, resetDataset } from '../../store/actionCreators'
11+
import { fileSize } from '../../helpers'
12+
import { saveAs } from 'file-saver'
13+
import MqEmpty from '../core/empty/MqEmpty'
14+
import MqJson from '../core/code/MqJson'
15+
import MqText from '../core/text/MqText'
16+
import React, { FunctionComponent, useEffect } from 'react'
17+
18+
interface DatasetColumnLineageProps {
19+
lineageDataset: LineageDataset
20+
}
21+
22+
interface StateProps {
23+
dataset: Dataset
24+
}
25+
26+
interface DispatchProps {
27+
fetchDataset: typeof fetchDataset
28+
resetDataset: typeof resetDataset
29+
}
30+
31+
type IProps = DatasetColumnLineageProps & DispatchProps & StateProps
32+
33+
const DatasetColumnLineage: FunctionComponent<IProps> = props => {
34+
const { dataset, lineageDataset, fetchDataset, resetDataset } = props
35+
const columnLineage = dataset.columnLineage
36+
37+
useEffect(() => {
38+
fetchDataset(lineageDataset.namespace, lineageDataset.name)
39+
}, [lineageDataset.name])
40+
41+
// unmounting
42+
useEffect(
43+
() => () => {
44+
resetDataset()
45+
},
46+
[]
47+
)
48+
49+
const handleDownloadPayload = (data: object) => {
50+
const title = `${lineageDataset.name}-${lineageDataset.namespace}-columnLineage`
51+
const blob = new Blob([JSON.stringify(data)], { type: 'application/json' })
52+
saveAs(blob, `${title}.json`)
53+
}
54+
55+
return (
56+
<>
57+
{columnLineage ? (
58+
<>
59+
{fileSize(JSON.stringify(columnLineage)).kiloBytes > 500 ? (
60+
<Box p={2}>
61+
<MqEmpty title={'Payload is too big for render'}>
62+
<div>
63+
<MqText subdued>Please click on button and download payload as file</MqText>
64+
<br />
65+
<Button
66+
variant='outlined'
67+
color='primary'
68+
onClick={() => handleDownloadPayload(columnLineage)}
69+
>
70+
Download payload
71+
</Button>
72+
</div>
73+
</MqEmpty>
74+
</Box>
75+
) : (
76+
<MqJson code={columnLineage} wrapLongLines={true} showLineNumbers={true} />
77+
)}
78+
</>
79+
) : (
80+
<MqEmpty
81+
title={'No column lineage'}
82+
body={'Column lineage not available for the specified dataset.'}
83+
/>
84+
)}
85+
</>
86+
)
87+
}
88+
89+
const mapStateToProps = (state: IState) => ({
90+
dataset: state.dataset.result
91+
})
92+
93+
const mapDispatchToProps = (dispatch: Redux.Dispatch) =>
94+
bindActionCreators(
95+
{
96+
fetchDataset: fetchDataset,
97+
resetDataset: resetDataset
98+
},
99+
dispatch
100+
)
101+
102+
export default connect(
103+
mapStateToProps,
104+
mapDispatchToProps
105+
)(DatasetColumnLineage)

‎web/src/components/datasets/DatasetDetailPage.tsx‎

Lines changed: 38 additions & 27 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,5 @@
11
// SPDX-License-Identifier: Apache-2.0
22

3-
import React, { ChangeEvent, FunctionComponent, SetStateAction, useEffect } from 'react'
4-
53
import * as Redux from 'redux'
64
import { Box, Chip, Tab, Tabs } from '@material-ui/core'
75
import { DatasetVersion } from '../../types/api'
@@ -15,14 +13,20 @@ import {
1513
import { LineageDataset } from '../lineage/types'
1614
import { bindActionCreators } from 'redux'
1715
import { connect } from 'react-redux'
18-
import { fetchDatasetVersions, resetDatasetVersions } from '../../store/actionCreators'
16+
import {
17+
fetchDatasetVersions,
18+
resetDataset,
19+
resetDatasetVersions
20+
} from '../../store/actionCreators'
1921
import { useHistory } from 'react-router-dom'
2022
import CircularProgress from '@material-ui/core/CircularProgress/CircularProgress'
2123
import CloseIcon from '@material-ui/icons/Close'
24+
import DatasetColumnLineage from './DatasetColumnLineage'
2225
import DatasetInfo from './DatasetInfo'
2326
import DatasetVersions from './DatasetVersions'
2427
import IconButton from '@material-ui/core/IconButton'
2528
import MqText from '../core/text/MqText'
29+
import React, { ChangeEvent, FunctionComponent, SetStateAction, useEffect } from 'react'
2630

2731
const styles = ({ spacing }: ITheme) => {
2832
return createStyles({
@@ -40,29 +44,20 @@ const styles = ({ spacing }: ITheme) => {
4044
'&:not(:last-of-type)': {
4145
marginRight: spacing(1)
4246
}
43-
},
44-
noData: {
45-
padding: '125px 0 0 0'
46-
},
47-
infoIcon: {
48-
paddingLeft: '3px',
49-
paddingTop: '3px'
50-
},
51-
updated: {
52-
marginTop: '10px'
5347
}
5448
})
5549
}
5650

5751
interface StateProps {
58-
dataset: LineageDataset
52+
lineageDataset: LineageDataset
5953
versions: DatasetVersion[]
6054
versionsLoading: boolean
6155
}
6256

6357
interface DispatchProps {
6458
fetchDatasetVersions: typeof fetchDatasetVersions
6559
resetDatasetVersions: typeof resetDatasetVersions
60+
resetDataset: typeof resetDataset
6661
}
6762

6863
type IProps = IWithStyles<typeof styles> & StateProps & DispatchProps
@@ -75,21 +70,30 @@ function a11yProps(index: number) {
7570
}
7671

7772
const DatasetDetailPage: FunctionComponent<IProps> = props => {
78-
const { classes, fetchDatasetVersions, resetDatasetVersions, versions, versionsLoading } = props
73+
const {
74+
classes,
75+
fetchDatasetVersions,
76+
resetDataset,
77+
resetDatasetVersions,
78+
versions,
79+
versionsLoading
80+
} = props
7981
const { root } = classes
8082
const history = useHistory()
8183
const i18next = require('i18next')
8284

8385
useEffect(() => {
84-
fetchDatasetVersions(props.dataset.namespace, props.dataset.name)
85-
}, [props.dataset.name])
86+
fetchDatasetVersions(props.lineageDataset.namespace, props.lineageDataset.name)
87+
}, [props.lineageDataset.name])
8688

8789
// unmounting
88-
useEffect(() => {
89-
return () => {
90+
useEffect(
91+
() => () => {
92+
resetDataset()
9093
resetDatasetVersions()
91-
}
92-
}, [])
94+
},
95+
[]
96+
)
9397

9498
const [tab, setTab] = React.useState(0)
9599
const handleChange = (event: ChangeEvent, newValue: SetStateAction<number>) => {
@@ -108,8 +112,8 @@ const DatasetDetailPage: FunctionComponent<IProps> = props => {
108112
return null
109113
}
110114

111-
const dataset = versions[0]
112-
const { name, tags, description } = dataset
115+
const firstVersion = versions[0]
116+
const { name, tags, description } = firstVersion
113117

114118
return (
115119
<Box my={2} className={root}>
@@ -136,6 +140,11 @@ const DatasetDetailPage: FunctionComponent<IProps> = props => {
136140
{...a11yProps(1)}
137141
disableRipple={true}
138142
/>
143+
<Tab
144+
label={i18next.t('datasets.column_lineage')}
145+
{...a11yProps(1)}
146+
disableRipple={true}
147+
/>
139148
</Tabs>
140149
</Box>
141150
<IconButton onClick={() => history.push('/datasets')}>
@@ -151,12 +160,13 @@ const DatasetDetailPage: FunctionComponent<IProps> = props => {
151160
</Box>
152161
{tab === 0 && (
153162
<DatasetInfo
154-
datasetFields={dataset.fields}
155-
facets={dataset.facets}
156-
run={dataset.createdByRun}
163+
datasetFields={firstVersion.fields}
164+
facets={firstVersion.facets}
165+
run={firstVersion.createdByRun}
157166
/>
158167
)}
159168
{tab === 1 && <DatasetVersions versions={props.versions} />}
169+
{tab === 2 && <DatasetColumnLineage lineageDataset={props.lineageDataset} />}
160170
</Box>
161171
)
162172
}
@@ -171,7 +181,8 @@ const mapDispatchToProps = (dispatch: Redux.Dispatch) =>
171181
bindActionCreators(
172182
{
173183
fetchDatasetVersions: fetchDatasetVersions,
174-
resetDatasetVersions: resetDatasetVersions
184+
resetDatasetVersions: resetDatasetVersions,
185+
resetDataset: resetDataset
175186
},
176187
dispatch
177188
)

‎web/src/components/sidenav/Sidenav.tsx‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -90,9 +90,9 @@ class Sidenav extends React.Component<SidenavProps> {
9090
title={i18next.t('sidenav.events')}
9191
active={this.props.location.pathname === '/events'}
9292
>
93-
<SVG
94-
src="https://raw.githubusercontent.com/MarquezProject/marquez/main/web/src/img/iconSearchArrow.svg"
95-
width={'30px'}
93+
<SVG
94+
src='https://raw.githubusercontent.com/MarquezProject/marquez/main/web/src/img/iconSearchArrow.svg'
95+
width={'30px'}
9696
/>
9797
</MqIconButton>
9898
</RouterLink>

‎web/src/i18n/config.ts‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -58,7 +58,8 @@ i18next
5858
},
5959
datasets: {
6060
latest_tab: 'LATEST SCHEMA',
61-
history_tab: 'VERSION HISTORY'
61+
history_tab: 'VERSION HISTORY',
62+
column_lineage: 'COLUMN LINEAGE'
6263
},
6364
datasets_route: {
6465
empty_title: 'No datasets found',

‎web/src/store/actionCreators/actionTypes.ts‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -22,8 +22,11 @@ export const RESET_JOBS = 'RESET_JOBS'
2222

2323
// datasets
2424
export const FETCH_DATASETS = 'FETCH_DATASETS'
25+
export const FETCH_DATASET = 'FETCH_DATASET'
2526
export const FETCH_DATASETS_SUCCESS = 'FETCH_DATASETS_SUCCESS'
27+
export const FETCH_DATASET_SUCCESS = 'FETCH_DATASET_SUCCESS'
2628
export const RESET_DATASETS = 'RESET_DATASETS'
29+
export const RESET_DATASET = 'RESET_DATASET'
2730
export const FETCH_DATASET_VERSIONS = 'FETCH_DATASET_VERSIONS'
2831
export const FETCH_DATASET_VERSIONS_SUCCESS = 'FETCH_DATASET_VERSIONS_SUCCESS'
2932
export const RESET_DATASET_VERSIONS = 'RESET_DATASET_VERSIONS'

‎web/src/store/actionCreators/index.ts‎

Lines changed: 19 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -48,6 +48,21 @@ export const fetchDatasetsSuccess = (datasets: Dataset[]) => ({
4848
}
4949
})
5050

51+
export const fetchDataset = (namespace: string, name: string) => ({
52+
type: actionTypes.FETCH_DATASET,
53+
payload: {
54+
namespace,
55+
name
56+
}
57+
})
58+
59+
export const fetchDatasetSuccess = (dataset: Dataset) => ({
60+
type: actionTypes.FETCH_DATASET_SUCCESS,
61+
payload: {
62+
dataset
63+
}
64+
})
65+
5166
export const fetchDatasetVersions = (namespace: string, name: string) => ({
5267
type: actionTypes.FETCH_DATASET_VERSIONS,
5368
payload: {
@@ -67,6 +82,10 @@ export const resetDatasetVersions = () => ({
6782
type: actionTypes.RESET_DATASET_VERSIONS
6883
})
6984

85+
export const resetDataset = () => ({
86+
type: actionTypes.RESET_DATASET
87+
})
88+
7089
export const resetDatasets = () => ({
7190
type: actionTypes.RESET_DATASETS
7291
})

‎web/src/store/reducers/dataset.ts‎

Lines changed: 26 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,26 @@
1+
// SPDX-License-Identifier: Apache-2.0
2+
3+
import { Dataset } from '../../types/api'
4+
import { FETCH_DATASET, FETCH_DATASET_SUCCESS, RESET_DATASET } from '../actionCreators/actionTypes'
5+
import { fetchDatasetSuccess } from '../actionCreators'
6+
7+
export type IDatasetState = { isLoading: boolean; result: Dataset; init: boolean }
8+
9+
export const initialState: IDatasetState = { isLoading: false, init: false, result: {} as Dataset }
10+
11+
type IDatasetAction = ReturnType<typeof fetchDatasetSuccess>
12+
13+
export default (state: IDatasetState = initialState, action: IDatasetAction): IDatasetState => {
14+
const { type, payload } = action
15+
16+
switch (type) {
17+
case FETCH_DATASET:
18+
return { ...state, isLoading: true }
19+
case FETCH_DATASET_SUCCESS:
20+
return { ...state, isLoading: false, init: true, result: payload.dataset }
21+
case RESET_DATASET:
22+
return initialState
23+
default:
24+
return state
25+
}
26+
}

‎web/src/store/reducers/index.ts‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,7 @@
33
import { History } from 'history'
44
import { Reducer, combineReducers } from 'redux'
55
import { connectRouter } from 'connected-react-router'
6+
import dataset, { IDatasetState } from './dataset'
67
import datasetVersions, { IDatasetVersionsState } from './datasetVersions'
78
import datasets, { IDatasetsState } from './datasets'
89
import display, { IDisplayState } from './display'
@@ -15,6 +16,7 @@ import search, { ISearchState } from './search'
1516

1617
export interface IState {
1718
datasets: IDatasetsState
19+
dataset: IDatasetState
1820
datasetVersions: IDatasetVersionsState
1921
events: IEventsState
2022
jobs: IJobsState
@@ -29,6 +31,7 @@ export interface IState {
2931
export default (history: History): Reducer =>
3032
combineReducers({
3133
router: connectRouter(history),
34+
dataset,
3235
datasets,
3336
datasetVersions,
3437
events,

0 commit comments

Comments
 (0)
⚡