Skip to content

Commit 639b371

Browse files
TobiGrTobiGr
authored
Merge pull request #1503 from AudricV/yt_fix-lockups-stream-items-parsing-in-some-cases
[YouTube] Fix extraction of playlist items and some properties in lockup view models
2 parents 389c728 + a6f1feb commit 639b371

12 files changed

Lines changed: 852 additions & 499 deletions

File tree

‎extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelTabExtractor.java‎

Lines changed: 2 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -393,13 +393,9 @@ private static void commitVideoLockup(@Nonnull final MultiInfoItemsCollector col
393393
@Nullable final String channelUrl) {
394394
collector.commit(
395395
new YoutubeStreamInfoItemLockupExtractor(lockupViewModel, timeAgoParser) {
396-
/**
397-
* Channel tabs use a 1-row metadata format [views, date]
398-
* instead of 2 rows [author][views, date].
399-
*/
400396
@Override
401-
protected int getInfoMetadataRowIndex() {
402-
return 0;
397+
protected boolean isChannelOrCoursePlaylistLockupItem() {
398+
return true;
403399
}
404400

405401
@Nonnull

‎extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubePlaylistExtractor.java‎

Lines changed: 105 additions & 58 deletions
Large diffs are not rendered by default.

‎extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamInfoItemLockupExtractor.java‎

Lines changed: 194 additions & 160 deletions
Large diffs are not rendered by default.

‎extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubePlaylistExtractorTest.java‎

Lines changed: 0 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -12,7 +12,6 @@
1212
import static org.schabi.newpipe.extractor.services.DefaultTests.defaultTestMoreItems;
1313
import static org.schabi.newpipe.extractor.services.DefaultTests.defaultTestRelatedItems;
1414

15-
import org.junit.jupiter.api.Disabled;
1615
import org.junit.jupiter.api.Test;
1716
import org.schabi.newpipe.extractor.ExtractorAsserts;
1817
import org.schabi.newpipe.extractor.ListExtractor;
@@ -28,7 +27,6 @@
2827
import org.schabi.newpipe.extractor.stream.ContentAvailability;
2928
import org.schabi.newpipe.extractor.utils.Utils;
3029

31-
import java.io.IOException;
3230
import java.util.List;
3331
import java.util.stream.Collectors;
3432

@@ -519,8 +517,6 @@ void testOnlyMembersOnlyVideos() throws Exception {
519517

520518
final List<StreamInfoItem> allItems = extractor.getInitialPage().getItems()
521519
.stream()
522-
.filter(StreamInfoItem.class::isInstance)
523-
.map(StreamInfoItem.class::cast)
524520
.collect(Collectors.toUnmodifiableList());
525521
final List<StreamInfoItem> membershipVideos = allItems.stream()
526522
.filter(item -> item.getContentAvailability() != ContentAvailability.MEMBERSHIP)
@@ -542,8 +538,6 @@ void uploaderName() throws Exception {
542538

543539
final List<StreamInfoItem> allItems = extractor.getInitialPage().getItems()
544540
.stream()
545-
.filter(StreamInfoItem.class::isInstance)
546-
.map(StreamInfoItem.class::cast)
547541
.collect(Collectors.toUnmodifiableList());
548542
assertEquals(14, allItems.size());
549543
assertEquals(extractor.getUploaderName(), allItems.get(0).getUploaderName());

‎extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeStreamInfoItemTest.java‎

Lines changed: 58 additions & 109 deletions
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,7 @@
77
import org.schabi.newpipe.extractor.localization.TimeAgoPatternsManager;
88
import org.schabi.newpipe.extractor.services.youtube.extractors.YoutubeStreamInfoItemExtractor;
99
import org.schabi.newpipe.extractor.services.youtube.extractors.YoutubeStreamInfoItemLockupExtractor;
10+
import org.schabi.newpipe.extractor.stream.ContentAvailability;
1011
import org.schabi.newpipe.extractor.stream.StreamType;
1112

1213
import java.io.FileInputStream;
@@ -53,7 +54,8 @@ void videoRendererPremiere() throws FileNotFoundException, JsonParserException {
5354
() -> assertEquals(-1, extractor.getViewCount()),
5455
() -> assertFalse(extractor.getThumbnails().isEmpty()),
5556
() -> assertEquals("Patience is key… MERCH SHOP : https://www.bluntbrosproductions.com Follow us on Instagram for early updates: ...", extractor.getShortDescription()),
56-
() -> assertFalse(extractor.isShortFormContent())
57+
() -> assertFalse(extractor.isShortFormContent()),
58+
() -> assertEquals(ContentAvailability.UPCOMING, extractor.getContentAvailability())
5759
);
5860
}
5961

@@ -82,7 +84,8 @@ void lockupViewModelPremiere()
8284
() -> assertEquals(-1, extractor.getViewCount()),
8385
() -> assertFalse(extractor.getThumbnails().isEmpty()),
8486
() -> assertNull(extractor.getShortDescription()),
85-
() -> assertFalse(extractor.isShortFormContent())
87+
() -> assertFalse(extractor.isShortFormContent()),
88+
() -> assertEquals(ContentAvailability.UPCOMING, extractor.getContentAvailability())
8689
);
8790
}
8891

@@ -99,15 +102,16 @@ void lockupViewModelVideo()
99102
() -> assertEquals("https://www.youtube.com/watch?v=dQw4w9WgXcQ", extractor.getUrl()),
100103
() -> assertEquals("VIDEO_TITLE", extractor.getName()),
101104
() -> assertEquals(974, extractor.getDuration()),
102-
() -> assertFalse(extractor.getThumbnails().isEmpty())
105+
() -> assertFalse(extractor.getThumbnails().isEmpty()),
106+
() -> assertEquals(ContentAvailability.AVAILABLE, extractor.getContentAvailability())
103107
);
104108
}
105109

106110
@Test
107-
void lockupViewModelLiveStream()
111+
void lockupViewModelLiveStreamNoViewer()
108112
throws FileNotFoundException, JsonParserException {
109113
final var json = JsonParser.object().from(new FileInputStream(getMockPath(
110-
YoutubeStreamInfoItemTest.class, "lockupViewModelLiveStream") + ".json"));
114+
YoutubeStreamInfoItemTest.class, "lockupViewModelLiveStreamNoViewer") + ".json"));
111115
final var timeAgoParser = TimeAgoPatternsManager.getTimeAgoParserFor(Localization.DEFAULT);
112116
final var extractor = new YoutubeStreamInfoItemLockupExtractor(json, timeAgoParser);
113117
assertAll(
@@ -119,7 +123,8 @@ void lockupViewModelLiveStream()
119123
() -> assertNull(extractor.getTextualUploadDate()),
120124
() -> assertNull(extractor.getUploadDate()),
121125
() -> assertEquals(0, extractor.getViewCount()),
122-
() -> assertFalse(extractor.getThumbnails().isEmpty())
126+
() -> assertFalse(extractor.getThumbnails().isEmpty()),
127+
() -> assertEquals(ContentAvailability.AVAILABLE, extractor.getContentAvailability())
123128
);
124129
}
125130

@@ -134,25 +139,24 @@ void lockupViewModelNoDuration()
134139
() -> assertEquals(StreamType.VIDEO_STREAM, extractor.getStreamType()),
135140
() -> assertFalse(extractor.isAd()),
136141
() -> assertEquals(-1, extractor.getDuration()),
137-
() -> assertFalse(extractor.getThumbnails().isEmpty())
142+
() -> assertFalse(extractor.getThumbnails().isEmpty()),
143+
() -> assertEquals(ContentAvailability.AVAILABLE, extractor.getContentAvailability())
138144
);
139145
}
140146

141147
/**
142-
* Tests that the info row search correctly extracts date and view count
143-
* from the 1-row channel format where parts are in normal order: [views, date].
148+
* Tests that the date and view count is extracted properly from the 1-row channel format.
144149
*/
145150
@Test
146-
void lockupViewModelOneRowNormal()
151+
void lockupViewModelChannelOneRow()
147152
throws FileNotFoundException, JsonParserException {
148153
final var json = JsonParser.object().from(new FileInputStream(getMockPath(
149-
YoutubeStreamInfoItemTest.class, "lockupViewModelOneRowNormal") + ".json"));
154+
YoutubeStreamInfoItemTest.class, "lockupViewModelChannelOneRow") + ".json"));
150155
final var timeAgoParser = TimeAgoPatternsManager.getTimeAgoParserFor(Localization.DEFAULT);
151156
final var extractor = new YoutubeStreamInfoItemLockupExtractor(json, timeAgoParser) {
152-
// Channel tabs use 1-row format at index 0
153157
@Override
154-
protected int getInfoMetadataRowIndex() {
155-
return 0;
158+
protected boolean isChannelOrCoursePlaylistLockupItem() {
159+
return true;
156160
}
157161
};
158162
assertAll(
@@ -161,52 +165,25 @@ protected int getInfoMetadataRowIndex() {
161165
() -> assertEquals("2 hours ago", extractor.getTextualUploadDate()),
162166
() -> assertNotNull(extractor.getUploadDate()),
163167
() -> assertEquals(3600000, extractor.getViewCount()), // 3.6m views
164-
() -> assertEquals(630, extractor.getDuration()) // 10:30
165-
);
166-
}
167-
168-
/**
169-
* Tests that the info row search correctly extracts date and view count
170-
* from the 1-row channel format where parts are in reversed order: [date, views].
171-
*/
172-
@Test
173-
void lockupViewModelOneRowReversed()
174-
throws FileNotFoundException, JsonParserException {
175-
final var json = JsonParser.object().from(new FileInputStream(getMockPath(
176-
YoutubeStreamInfoItemTest.class, "lockupViewModelOneRowReversed") + ".json"));
177-
final var timeAgoParser = TimeAgoPatternsManager.getTimeAgoParserFor(Localization.DEFAULT);
178-
final var extractor = new YoutubeStreamInfoItemLockupExtractor(json, timeAgoParser) {
179-
// Channel tabs use 1-row format at index 0
180-
@Override
181-
protected int getInfoMetadataRowIndex() {
182-
return 0;
183-
}
184-
};
185-
assertAll(
186-
() -> assertEquals(StreamType.VIDEO_STREAM, extractor.getStreamType()),
187-
() -> assertEquals("Test Video One Row Reversed", extractor.getName()),
188-
() -> assertEquals("1 day ago", extractor.getTextualUploadDate()),
189-
() -> assertNotNull(extractor.getUploadDate()),
190-
() -> assertEquals(1200, extractor.getViewCount()), // 1.2K views
191-
() -> assertEquals(300, extractor.getDuration()) // 5:00
168+
() -> assertEquals(630, extractor.getDuration()), // 10:30
169+
() -> assertEquals(ContentAvailability.AVAILABLE, extractor.getContentAvailability())
192170
);
193171
}
194172

195173
/**
196-
* Tests that the info row search handles 1-row format with only view count
197-
* (no date text present) - e.g. for livestreams with watching count only.
174+
* Tests that the info row search handles 1-row format with only view count (no date text
175+
* present), returned for livestreams on channels.
198176
*/
199177
@Test
200-
void lockupViewModelOneRowViewsOnly()
178+
void lockupViewModelChannelOneRowLiveViewsOnly()
201179
throws FileNotFoundException, JsonParserException {
202180
final var json = JsonParser.object().from(new FileInputStream(getMockPath(
203-
YoutubeStreamInfoItemTest.class, "lockupViewModelOneRowViewsOnly") + ".json"));
181+
YoutubeStreamInfoItemTest.class, "lockupViewModelChannelOneRowLiveViewsOnly") + ".json"));
204182
final var timeAgoParser = TimeAgoPatternsManager.getTimeAgoParserFor(Localization.DEFAULT);
205183
final var extractor = new YoutubeStreamInfoItemLockupExtractor(json, timeAgoParser) {
206-
// Channel tabs use 1-row format at index 0
207184
@Override
208-
protected int getInfoMetadataRowIndex() {
209-
return 0;
185+
protected boolean isChannelOrCoursePlaylistLockupItem() {
186+
return true;
210187
}
211188
};
212189
assertAll(
@@ -215,67 +192,38 @@ protected int getInfoMetadataRowIndex() {
215192
() -> assertNull(extractor.getTextualUploadDate()),
216193
() -> assertNull(extractor.getUploadDate()),
217194
() -> assertEquals(500, extractor.getViewCount()), // 500 watching
218-
() -> assertEquals(-1, extractor.getDuration())
195+
() -> assertEquals(-1, extractor.getDuration()),
196+
() -> assertEquals(ContentAvailability.AVAILABLE, extractor.getContentAvailability())
219197
);
220198
}
221199

222-
/**
223-
* Tests that channel tab items with non-views text in the metadata row
224-
* (e.g. section headers) don't crash and return -1 for views.
225-
* Regression test for blind metadataPart(infoRowIndex, 0) fallback.
226-
*/
227200
@Test
228-
void lockupViewModelChannelTabSectionHeader()
201+
void lockupViewModelChannelMembersOnly()
229202
throws FileNotFoundException, JsonParserException {
230203
final var json = JsonParser.object().from(new FileInputStream(getMockPath(
231-
YoutubeStreamInfoItemTest.class, "lockupViewModelChannelTabSectionHeader")
204+
YoutubeStreamInfoItemTest.class, "lockupviewmodelchannelmembersonly")
232205
+ ".json"));
233206
final var timeAgoParser = TimeAgoPatternsManager.getTimeAgoParserFor(
234207
Localization.DEFAULT);
235208
final var extractor = new YoutubeStreamInfoItemLockupExtractor(json, timeAgoParser) {
236-
// Channel tabs use 1-row format at index 0
237209
@Override
238-
protected int getInfoMetadataRowIndex() {
239-
return 0;
210+
protected boolean isChannelOrCoursePlaylistLockupItem() {
211+
return true;
240212
}
241213
};
242214
assertAll(
243215
() -> assertEquals(StreamType.VIDEO_STREAM, extractor.getStreamType()),
244-
() -> assertEquals("Section Header Video", extractor.getName()),
245-
() -> assertNull(extractor.getTextualUploadDate()),
246-
() -> assertNull(extractor.getUploadDate()),
216+
() -> assertFalse(extractor.isAd()),
217+
() -> assertEquals("https://www.youtube.com/watch?v=x3pS1_qqtIs", extractor.getUrl()),
218+
() -> assertEquals("Daily Linux News - S03E105 - EU will fund European datacenters, Steam Deck's massive price increase", extractor.getName()),
219+
() -> assertEquals(725, extractor.getDuration()),
220+
() -> assertEquals("12 hours ago", extractor.getTextualUploadDate()),
221+
() -> assertNotNull(extractor.getUploadDate()),
247222
() -> assertEquals(-1, extractor.getViewCount()),
248-
() -> assertEquals(-1, extractor.getDuration())
249-
);
250-
}
251-
252-
/**
253-
* Tests that live streams in channel tabs with only a channel name row
254-
* (no viewer count) don't crash and return 0 for views.
255-
* Regression test for blind metadataPart(infoRowIndex, 0) fallback.
256-
*/
257-
@Test
258-
void lockupViewModelChannelTabLiveNoViewers()
259-
throws FileNotFoundException, JsonParserException {
260-
final var json = JsonParser.object().from(new FileInputStream(getMockPath(
261-
YoutubeStreamInfoItemTest.class, "lockupViewModelChannelTabLiveNoViewers")
262-
+ ".json"));
263-
final var timeAgoParser = TimeAgoPatternsManager.getTimeAgoParserFor(
264-
Localization.DEFAULT);
265-
final var extractor = new YoutubeStreamInfoItemLockupExtractor(json, timeAgoParser) {
266-
// Channel tabs use 1-row format at index 0
267-
@Override
268-
protected int getInfoMetadataRowIndex() {
269-
return 0;
270-
}
271-
};
272-
assertAll(
273-
() -> assertEquals(StreamType.LIVE_STREAM, extractor.getStreamType()),
274-
() -> assertEquals("Live Stream No Viewers", extractor.getName()),
275-
() -> assertNull(extractor.getTextualUploadDate()),
276-
() -> assertNull(extractor.getUploadDate()),
277-
() -> assertEquals(0, extractor.getViewCount()),
278-
() -> assertEquals(-1, extractor.getDuration())
223+
() -> assertFalse(extractor.getThumbnails().isEmpty()),
224+
() -> assertNull(extractor.getShortDescription()),
225+
() -> assertFalse(extractor.isShortFormContent()),
226+
() -> assertEquals(ContentAvailability.MEMBERSHIP, extractor.getContentAvailability())
279227
);
280228
}
281229

@@ -286,21 +234,22 @@ void emptyTitle() throws FileNotFoundException, JsonParserException {
286234
final var timeAgoParser = TimeAgoPatternsManager.getTimeAgoParserFor(Localization.DEFAULT);
287235
final var extractor = new YoutubeStreamInfoItemExtractor(json, timeAgoParser);
288236
assertAll(
289-
() -> assertEquals(StreamType.VIDEO_STREAM, extractor.getStreamType()),
290-
() -> assertFalse(extractor.isAd()),
291-
() -> assertEquals("https://www.youtube.com/watch?v=nc1kN8ZSfGQ", extractor.getUrl()),
292-
() -> assertEquals("", extractor.getName()),
293-
() -> assertEquals(39, extractor.getDuration()),
294-
() -> assertEquals("hyper", extractor.getUploaderName()),
295-
() -> assertEquals("https://www.youtube.com/channel/UCSezUnbvCLYBXuUlPcXU_QQ", extractor.getUploaderUrl()),
296-
() -> assertFalse(extractor.getUploaderAvatars().isEmpty()),
297-
() -> assertTrue(extractor.isUploaderVerified()),
298-
() -> assertEquals("8 years ago", extractor.getTextualUploadDate()),
299-
() -> assertNotNull(extractor.getUploadDate()),
300-
() -> assertTrue(extractor.getViewCount() >= 1318193),
301-
() -> assertFalse(extractor.getThumbnails().isEmpty()),
302-
() -> assertNull(extractor.getShortDescription()),
303-
() -> assertFalse(extractor.isShortFormContent())
237+
() -> assertEquals(StreamType.VIDEO_STREAM, extractor.getStreamType()),
238+
() -> assertFalse(extractor.isAd()),
239+
() -> assertEquals("https://www.youtube.com/watch?v=nc1kN8ZSfGQ", extractor.getUrl()),
240+
() -> assertEquals("", extractor.getName()),
241+
() -> assertEquals(39, extractor.getDuration()),
242+
() -> assertEquals("hyper", extractor.getUploaderName()),
243+
() -> assertEquals("https://www.youtube.com/channel/UCSezUnbvCLYBXuUlPcXU_QQ", extractor.getUploaderUrl()),
244+
() -> assertFalse(extractor.getUploaderAvatars().isEmpty()),
245+
() -> assertTrue(extractor.isUploaderVerified()),
246+
() -> assertEquals("8 years ago", extractor.getTextualUploadDate()),
247+
() -> assertNotNull(extractor.getUploadDate()),
248+
() -> assertTrue(extractor.getViewCount() >= 1318193),
249+
() -> assertFalse(extractor.getThumbnails().isEmpty()),
250+
() -> assertNull(extractor.getShortDescription()),
251+
() -> assertFalse(extractor.isShortFormContent()),
252+
() -> assertEquals(ContentAvailability.AVAILABLE, extractor.getContentAvailability())
304253
);
305254
}
306255
}

0 commit comments

Comments
 (0)
⚡