Skip to content

Commit e18c13e

Browse files
pawel-big-lebowskipawel-big-lebowski
committed
rename ColumnLevelLineage -> ColumnLineage
Signed-off-by: Pawel Leszczynski <leszczynski.pawel@gmail.com>
1 parent 1aefca2 commit e18c13e

8 files changed

Lines changed: 47 additions & 48 deletions

File tree

‎api/src/main/java/marquez/db/BaseDao.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -55,5 +55,5 @@ public interface BaseDao extends SqlObject {
5555
OpenLineageDao createOpenLineageDao();
5656

5757
@CreateSqlObject
58-
ColumnLevelLineageDao createColumnLevelLineageDao();
58+
ColumnLineageDao createColumnLineageDao();
5959
}

api/src/main/java/marquez/db/ColumnLevelLineageDao.java renamed to api/src/main/java/marquez/db/ColumnLineageDao.java

Lines changed: 14 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -10,18 +10,18 @@
1010
import java.util.List;
1111
import java.util.UUID;
1212
import java.util.stream.Collectors;
13-
import marquez.db.mappers.ColumnLevelLineageRowMapper;
14-
import marquez.db.models.ColumnLevelLineageRow;
13+
import marquez.db.mappers.ColumnLineageRowMapper;
14+
import marquez.db.models.ColumnLineageRow;
1515
import org.apache.commons.lang3.tuple.Pair;
1616
import org.jdbi.v3.sqlobject.config.RegisterRowMapper;
1717
import org.jdbi.v3.sqlobject.customizer.BindBeanList;
1818
import org.jdbi.v3.sqlobject.statement.SqlQuery;
1919
import org.jdbi.v3.sqlobject.statement.SqlUpdate;
2020

21-
@RegisterRowMapper(ColumnLevelLineageRowMapper.class)
22-
public interface ColumnLevelLineageDao extends BaseDao {
21+
@RegisterRowMapper(ColumnLineageRowMapper.class)
22+
public interface ColumnLineageDao extends BaseDao {
2323

24-
default List<ColumnLevelLineageRow> upsertColumnLevelLineageRow(
24+
default List<ColumnLineageRow> upsertColumnLineageRow(
2525
UUID outputDatasetVersionUuid,
2626
UUID outputDatasetFieldUuid,
2727
List<Pair<UUID, UUID>> inputs,
@@ -33,11 +33,11 @@ default List<ColumnLevelLineageRow> upsertColumnLevelLineageRow(
3333
return Collections.emptyList();
3434
}
3535

36-
List<ColumnLevelLineageRow> rows =
36+
List<ColumnLineageRow> rows =
3737
inputs.stream()
3838
.map(
3939
input ->
40-
new ColumnLevelLineageRow(
40+
new ColumnLineageRow(
4141
outputDatasetVersionUuid,
4242
outputDatasetFieldUuid,
4343
input.getLeft(), // input_dataset_version_uuid
@@ -47,19 +47,19 @@ default List<ColumnLevelLineageRow> upsertColumnLevelLineageRow(
4747
now,
4848
now))
4949
.collect(Collectors.toList());
50-
doUpsertColumnLevelLineageRow(rows.toArray(new ColumnLevelLineageRow[0]));
51-
return findColumnLevelLineageByDatasetVersionColumnAndOutputDatasetField(
50+
doUpsertColumnLineageRow(rows.toArray(new ColumnLineageRow[0]));
51+
return findColumnLineageByDatasetVersionColumnAndOutputDatasetField(
5252
outputDatasetVersionUuid, outputDatasetFieldUuid);
5353
}
5454

5555
@SqlQuery(
56-
"SELECT * FROM column_level_lineage WHERE output_dataset_version_uuid = :datasetVersionUuid AND output_dataset_field_uuid = :outputDatasetFieldUuid")
57-
List<ColumnLevelLineageRow> findColumnLevelLineageByDatasetVersionColumnAndOutputDatasetField(
56+
"SELECT * FROM column_lineage WHERE output_dataset_version_uuid = :datasetVersionUuid AND output_dataset_field_uuid = :outputDatasetFieldUuid")
57+
List<ColumnLineageRow> findColumnLineageByDatasetVersionColumnAndOutputDatasetField(
5858
UUID datasetVersionUuid, UUID outputDatasetFieldUuid);
5959

6060
@SqlUpdate(
6161
"""
62-
INSERT INTO column_level_lineage (
62+
INSERT INTO column_lineage (
6363
output_dataset_version_uuid,
6464
output_dataset_field_uuid,
6565
input_dataset_version_uuid,
@@ -74,9 +74,8 @@ ON CONFLICT (output_dataset_version_uuid, output_dataset_field_uuid, input_datas
7474
transformation_description = EXCLUDED.transformation_description,
7575
transformation_type = EXCLUDED.transformation_type,
7676
updated_at = EXCLUDED.updated_at
77-
RETURNING *
7877
""")
79-
void doUpsertColumnLevelLineageRow(
78+
void doUpsertColumnLineageRow(
8079
@BindBeanList(
8180
propertyNames = {
8281
"outputDatasetVersionUuid",
@@ -89,5 +88,5 @@ void doUpsertColumnLevelLineageRow(
8988
"updatedAt"
9089
},
9190
value = "values")
92-
ColumnLevelLineageRow... rows);
91+
ColumnLineageRow... rows);
9392
}

‎api/src/main/java/marquez/db/OpenLineageDao.java‎

Lines changed: 11 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -34,7 +34,7 @@
3434
import marquez.db.DatasetFieldDao.DatasetFieldMapping;
3535
import marquez.db.JobVersionDao.BagOfJobVersionInfo;
3636
import marquez.db.mappers.LineageEventMapper;
37-
import marquez.db.models.ColumnLevelLineageRow;
37+
import marquez.db.models.ColumnLineageRow;
3838
import marquez.db.models.DatasetFieldRow;
3939
import marquez.db.models.DatasetRow;
4040
import marquez.db.models.DatasetSymlinkRow;
@@ -137,7 +137,7 @@ default UpdateLineageRow updateBaseMarquezModel(LineageEvent event, ObjectMapper
137137
RunDao runDao = createRunDao();
138138
RunArgsDao runArgsDao = createRunArgsDao();
139139
RunStateDao runStateDao = createRunStateDao();
140-
ColumnLevelLineageDao columnLevelLineageDao = createColumnLevelLineageDao();
140+
ColumnLineageDao columnLineageDao = createColumnLineageDao();
141141

142142
Instant now = event.getEventTime().withZoneSameInstant(ZoneId.of("UTC")).toInstant();
143143

@@ -331,7 +331,7 @@ default UpdateLineageRow updateBaseMarquezModel(LineageEvent event, ObjectMapper
331331
datasetVersionDao,
332332
datasetFieldDao,
333333
runDao,
334-
columnLevelLineageDao);
334+
columnLineageDao);
335335
datasetInputs.add(record);
336336
}
337337
}
@@ -354,7 +354,7 @@ default UpdateLineageRow updateBaseMarquezModel(LineageEvent event, ObjectMapper
354354
datasetVersionDao,
355355
datasetFieldDao,
356356
runDao,
357-
columnLevelLineageDao);
357+
columnLineageDao);
358358
datasetOutputs.add(record);
359359
}
360360
}
@@ -551,7 +551,7 @@ default DatasetRecord upsertLineageDataset(
551551
DatasetVersionDao datasetVersionDao,
552552
DatasetFieldDao datasetFieldDao,
553553
RunDao runDao,
554-
ColumnLevelLineageDao columnLevelLineageDao) {
554+
ColumnLineageDao columnLineageDao) {
555555
NamespaceRow dsNamespace =
556556
namespaceDao.upsertNamespaceRow(
557557
UUID.randomUUID(), now, ds.getNamespace(), DEFAULT_NAMESPACE_OWNER);
@@ -702,28 +702,28 @@ default DatasetRecord upsertLineageDataset(
702702
}
703703
}
704704

705-
List<ColumnLevelLineageRow> columnLineageRows = Collections.emptyList();
705+
List<ColumnLineageRow> columnLineageRows = Collections.emptyList();
706706
if (!isInput) {
707707
columnLineageRows =
708708
upsertColumnLineage(
709709
runUuid,
710710
ds,
711711
now,
712712
datasetFields,
713-
columnLevelLineageDao,
713+
columnLineageDao,
714714
datasetFieldDao,
715715
datasetVersionRow);
716716
}
717717

718718
return new DatasetRecord(datasetRow, datasetVersionRow, datasetNamespace, columnLineageRows);
719719
}
720720

721-
private List<ColumnLevelLineageRow> upsertColumnLineage(
721+
private List<ColumnLineageRow> upsertColumnLineage(
722722
UUID runUuid,
723723
Dataset ds,
724724
Instant now,
725725
List<DatasetFieldRow> datasetFields,
726-
ColumnLevelLineageDao columnLevelLineageDao,
726+
ColumnLineageDao columnLineageDao,
727727
DatasetFieldDao datasetFieldDao,
728728
DatasetVersionRow datasetVersionRow) {
729729
// get all the fields related to this particular run
@@ -770,8 +770,8 @@ private List<ColumnLevelLineageRow> upsertColumnLineage(
770770
fieldData.getDatasetFieldUuid()))
771771
.collect(Collectors.toList());
772772

773-
return columnLevelLineageDao
774-
.upsertColumnLevelLineageRow(
773+
return columnLineageDao
774+
.upsertColumnLineageRow(
775775
datasetVersionRow.getUuid(),
776776
outputField.get().getUuid(),
777777
inputFields,

api/src/main/java/marquez/db/mappers/ColumnLevelLineageRowMapper.java renamed to api/src/main/java/marquez/db/mappers/ColumnLineageRowMapper.java

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -15,16 +15,16 @@
1515
import java.sql.SQLException;
1616
import lombok.NonNull;
1717
import marquez.db.Columns;
18-
import marquez.db.models.ColumnLevelLineageRow;
18+
import marquez.db.models.ColumnLineageRow;
1919
import org.jdbi.v3.core.mapper.RowMapper;
2020
import org.jdbi.v3.core.statement.StatementContext;
2121

22-
public class ColumnLevelLineageRowMapper implements RowMapper<ColumnLevelLineageRow> {
22+
public class ColumnLineageRowMapper implements RowMapper<ColumnLineageRow> {
2323

2424
@Override
25-
public ColumnLevelLineageRow map(@NonNull ResultSet results, @NonNull StatementContext context)
25+
public ColumnLineageRow map(@NonNull ResultSet results, @NonNull StatementContext context)
2626
throws SQLException {
27-
return new ColumnLevelLineageRow(
27+
return new ColumnLineageRow(
2828
uuidOrThrow(results, Columns.OUTPUT_DATASET_VERSION_UUID),
2929
uuidOrThrow(results, Columns.OUTPUT_DATASET_FIELD_UUID),
3030
uuidOrThrow(results, Columns.INPUT_DATASET_VERSION_UUID),

api/src/main/java/marquez/db/models/ColumnLevelLineageRow.java renamed to api/src/main/java/marquez/db/models/ColumnLineageRow.java

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -16,7 +16,7 @@
1616
@AllArgsConstructor
1717
@EqualsAndHashCode
1818
@ToString
19-
public class ColumnLevelLineageRow {
19+
public class ColumnLineageRow {
2020
@Getter @NonNull private final UUID outputDatasetVersionUuid;
2121
@Getter @NonNull private final UUID outputDatasetFieldUuid;
2222
@Getter @NonNull private final UUID inputDatasetVersionUuid;

‎api/src/main/java/marquez/db/models/UpdateLineageRow.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -30,6 +30,6 @@ public static class DatasetRecord {
3030
DatasetRow datasetRow;
3131
DatasetVersionRow datasetVersionRow;
3232
NamespaceRow namespaceRow;
33-
List<ColumnLevelLineageRow> columnLineageRows;
33+
List<ColumnLineageRow> columnLineageRows;
3434
}
3535
}

api/src/main/resources/marquez/db/migration/V49__column_level_lineage.sql renamed to api/src/main/resources/marquez/db/migration/V49__column_lineage.sql

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
/* SPDX-License-Identifier: Apache-2.0 */
22

3-
CREATE TABLE column_level_lineage (
3+
CREATE TABLE column_lineage (
44
output_dataset_version_uuid uuid REFERENCES dataset_versions(uuid), -- allows join to run_id
55
output_dataset_field_uuid uuid REFERENCES dataset_fields(uuid),
66
input_dataset_version_uuid uuid REFERENCES dataset_versions(uuid), -- speed up graph column lineage graph traversal

api/src/test/java/marquez/db/ColumnLevelLineageDaoTest.java renamed to api/src/test/java/marquez/db/ColumnLineageDaoTest.java

Lines changed: 14 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -15,7 +15,7 @@
1515
import java.util.List;
1616
import java.util.UUID;
1717
import marquez.common.models.DatasetType;
18-
import marquez.db.models.ColumnLevelLineageRow;
18+
import marquez.db.models.ColumnLineageRow;
1919
import marquez.db.models.DatasetRow;
2020
import marquez.db.models.DatasetVersionRow;
2121
import marquez.db.models.NamespaceRow;
@@ -30,9 +30,9 @@
3030
import org.junit.jupiter.api.extension.ExtendWith;
3131

3232
@ExtendWith(MarquezJdbiExternalPostgresExtension.class)
33-
public class ColumnLevelLineageDaoTest {
33+
public class ColumnLineageDaoTest {
3434

35-
private static ColumnLevelLineageDao dao;
35+
private static ColumnLineageDao dao;
3636
private static DatasetFieldDao fieldDao;
3737
private static DatasetDao datasetDao;
3838
private static NamespaceDao namespaceDao;
@@ -50,7 +50,7 @@ public class ColumnLevelLineageDaoTest {
5050

5151
@BeforeAll
5252
public static void setUpOnce(Jdbi jdbi) {
53-
dao = jdbi.onDemand(ColumnLevelLineageDao.class);
53+
dao = jdbi.onDemand(ColumnLineageDao.class);
5454
fieldDao = jdbi.onDemand(DatasetFieldDao.class);
5555
datasetDao = jdbi.onDemand(DatasetDao.class);
5656
namespaceDao = jdbi.onDemand(NamespaceDao.class);
@@ -73,7 +73,7 @@ public void setup() {
7373
"",
7474
sourceRow.getUuid(),
7575
"",
76-
"",
76+
"inputDataset",
7777
"",
7878
"",
7979
false);
@@ -86,7 +86,7 @@ public void setup() {
8686
"",
8787
sourceRow.getUuid(),
8888
"",
89-
"",
89+
"outputDataset",
9090
"",
9191
"",
9292
false);
@@ -135,7 +135,7 @@ public void setup() {
135135
public void tearDown(Jdbi jdbi) {
136136
jdbi.inTransaction(
137137
handle -> {
138-
handle.execute("DELETE FROM column_level_lineage");
138+
handle.execute("DELETE FROM column_lineage");
139139
handle.execute("DELETE FROM dataset_versions");
140140
handle.execute("DELETE FROM dataset_fields");
141141
handle.execute("DELETE FROM datasets");
@@ -154,8 +154,8 @@ void testUpsertMultipleColumns() {
154154
fieldDao.upsert(inputFieldUuid1, now, "a", "string", "desc", inputDatasetRow.getUuid());
155155
fieldDao.upsert(inputFieldUuid2, now, "b", "string", "desc", inputDatasetRow.getUuid());
156156

157-
List<ColumnLevelLineageRow> rows =
158-
dao.upsertColumnLevelLineageRow(
157+
List<ColumnLineageRow> rows =
158+
dao.upsertColumnLineageRow(
159159
outputDatasetVersionRow.getUuid(),
160160
outputDatasetFieldUuid,
161161
Arrays.asList(
@@ -180,8 +180,8 @@ void testUpsertMultipleColumns() {
180180

181181
@Test
182182
void testUpsertEmptyList() {
183-
List<ColumnLevelLineageRow> rows =
184-
dao.upsertColumnLevelLineageRow(
183+
List<ColumnLineageRow> rows =
184+
dao.upsertColumnLineageRow(
185185
UUID.randomUUID(),
186186
outputDatasetFieldUuid,
187187
Collections.emptyList(), // provide empty list
@@ -198,15 +198,15 @@ void testUpsertOnUpdatePreventsDuplicates() {
198198
UUID inputFieldUuid = UUID.randomUUID();
199199
fieldDao.upsert(inputFieldUuid, now, "a", "string", "desc", inputDatasetRow.getUuid());
200200

201-
dao.upsertColumnLevelLineageRow(
201+
dao.upsertColumnLineageRow(
202202
inputDatasetVersionRow.getUuid(),
203203
outputDatasetFieldUuid,
204204
Arrays.asList(Pair.of(inputDatasetVersionRow.getUuid(), inputFieldUuid)),
205205
transformationDescription,
206206
transformationType,
207207
now);
208-
List<ColumnLevelLineageRow> rows =
209-
dao.upsertColumnLevelLineageRow(
208+
List<ColumnLineageRow> rows =
209+
dao.upsertColumnLineageRow(
210210
inputDatasetVersionRow.getUuid(),
211211
outputDatasetFieldUuid,
212212
Arrays.asList(Pair.of(inputDatasetVersionRow.getUuid(), inputFieldUuid)),

0 commit comments

Comments
 (0)
⚡