Skip to content

Commit 2fab706

Browse files
flynnplattflynnplatt
committed
make STAC prefetch async so that large catalogs don't take forever
1 parent 3665402 commit 2fab706

File tree

2 files changed

+144
-112
lines changed

2 files changed

+144
-112
lines changed

‎src/essence/Basics/Layers_/Layers_.js‎

Lines changed: 138 additions & 106 deletions
Original file line numberDiff line numberDiff line change
@@ -78,8 +78,8 @@ const L_ = {
7878
_layersBeingMade: {},
7979
_onLoadCallbacks: [],
8080
_loaded: false,
81-
init: function (configData, missionsList, urlOnLayers) {
82-
parseConfig(configData, urlOnLayers)
81+
init: async function (configData, missionsList, urlOnLayers) {
82+
await parseConfig(configData, urlOnLayers)
8383
L_.missionsList = missionsList
8484
},
8585
onceLoaded(cb) {
@@ -2994,7 +2994,7 @@ const L_ = {
29942994
},
29952995
parseConfig: parseConfig,
29962996

2997-
resetConfig: function (data) {
2997+
resetConfig: async function (data) {
29982998
// Save so we can make sure we reproduce the same layer settings after parsing the config
29992999
const toggledArray = { ...L_.layers.on }
30003000

@@ -3006,7 +3006,7 @@ const L_ = {
30063006
L_.layers.dataFlat = []
30073007
L_._layersLoaded = []
30083008

3009-
L_.parseConfig(data)
3009+
await L_.parseConfig(data)
30103010

30113011
// Set back
30123012
L_.layers.on = { ...L_.layers.on, ...toggledArray }
@@ -3137,7 +3137,7 @@ const L_ = {
31373137
// If we have a few changes waiting in the queue, we only need to parse the config once
31383138
// as the last item in the queue should have the latest data
31393139
const lastLayer = layerQueueList[layerQueueList.length - 1]
3140-
L_.resetConfig(lastLayer.data)
3140+
await L_.resetConfig(lastLayer.data)
31413141

31423142
while (layerQueueList.length > 0) {
31433143
const firstLayer = layerQueueList.shift()
@@ -3551,7 +3551,7 @@ const L_ = {
35513551

35523552
//Takes in a configData object and does a depth-first search through its
35533553
// layers and sets L_ variables
3554-
function parseConfig(configData, urlOnLayers) {
3554+
async function parseConfig(configData, urlOnLayers) {
35553555
//Create parsed configData
35563556
L_.configData = configData
35573557

@@ -3620,17 +3620,17 @@ function parseConfig(configData, urlOnLayers) {
36203620
const layers = L_.configData.layers
36213621

36223622
//Begin recursively going through those layers
3623-
expandLayers(layers, 0, null)
3623+
await expandLayers(layers, 0, null)
36243624

3625-
function expandLayers(d, level, prevName) {
3625+
async function expandLayers(d, level, prevName) {
36263626
const stacRegex = /^stac(-((item)|(catalog)|(collection)))?:/i
36273627

36283628
//Iterate over each layer
36293629
for (let i = 0; i < d.length; i++) {
36303630
// check if this is a vector STAC catalog or collection
36313631
// if so, prefetch the data and replace this entry
36323632
if (d[i].type === 'vector' && stacRegex.test(d[i].url)) {
3633-
d[i] = getSTACLayers(d[i])
3633+
d[i] = await getSTACLayers(d[i])
36343634
}
36353635

36363636
// Quick hack to use uuid instead of name as main id
@@ -3806,109 +3806,141 @@ function parseConfig(configData, urlOnLayers) {
38063806

38073807
// recurse through a STAC layer building sublayers
38083808
function getSTACLayers(d) {
3809-
let stac_data
3810-
const stacRegex =
3811-
/^(?<prefix>stac(-((item)|(catalog)|(collection)))?:)?(?<url>.*)/i
3812-
const urlMatch = d.url.match(stacRegex)
3813-
if (!urlMatch) {
3814-
console.warn('Could not process STAC URL')
3815-
return d
3816-
}
3817-
const { prefix, url } = urlMatch.groups
3818-
d.url = url // replace the current URL so we no longer need to worry about the special prefix
3819-
if (prefix !== 'stac-item:') {
3820-
$.ajax({
3821-
url: L_.getUrl('stac', d.url, d),
3822-
success: (resp) => {
3823-
stac_data = resp
3824-
},
3825-
async: false,
3826-
})
3827-
const path = d.url.split('/').slice(0, -1).join('/')
3828-
const basename = F_.fileNameFromPath(d.url)
3829-
const stac_type = stac_data.type.toLowerCase()
3830-
if (stac_type === 'catalog') {
3831-
const sublayers = []
3832-
const children = stac_data.links.filter((l) =>
3833-
/^child/i.test(l.rel)
3834-
)
3835-
for (let i = 0; i < children.length; i++) {
3836-
const uuid = `${d.uuid}-${i}`
3837-
sublayers.push(
3838-
getSTACLayers(
3839-
Object.assign({}, d, {
3840-
url: children[i].href.replace('./', `${path}/`),
3841-
display_name:
3842-
children[i].title ||
3843-
F_.fileNameFromPath(children[i].href),
3844-
uuid: uuid,
3845-
name: uuid,
3846-
})
3847-
)
3848-
)
3849-
}
3809+
return new Promise(async (resolve, reject) => {
3810+
let stac_data
3811+
const stacRegex =
3812+
/^(?<prefix>stac(-((item)|(catalog)|(collection)))?:)?(?<url>.*)/i
3813+
const urlMatch = d.url.match(stacRegex)
3814+
if (!urlMatch) {
3815+
console.warn('Could not process STAC URL')
3816+
resolve(d)
3817+
}
3818+
const { prefix, url } = urlMatch.groups
3819+
d.url = url // replace the current URL so we no longer need to worry about the special prefix
3820+
if (prefix !== 'stac-item:') {
3821+
$.ajax({
3822+
url: L_.getUrl('stac', d.url, d),
3823+
success: async (resp) => {
3824+
stac_data = resp
3825+
const path = d.url.split('/').slice(0, -1).join('/')
3826+
const basename = F_.fileNameFromPath(d.url)
3827+
const stac_type = stac_data.type.toLowerCase()
3828+
if (stac_type === 'catalog') {
3829+
let sublayers = []
3830+
const children = stac_data.links.filter((l) =>
3831+
/^child/i.test(l.rel)
3832+
)
3833+
const promArr = []
3834+
for (let i = 0; i < children.length; i++) {
3835+
const uuid = `${d.uuid}-${i}`
3836+
promArr.push(
3837+
getSTACLayers(
3838+
Object.assign({}, d, {
3839+
url: children[i].href.replace(
3840+
'./',
3841+
`${path}/`
3842+
),
3843+
display_name:
3844+
children[i].title ||
3845+
F_.fileNameFromPath(
3846+
children[i].href
3847+
),
3848+
uuid: uuid,
3849+
name: uuid,
3850+
})
3851+
)
3852+
)
3853+
}
38503854

3851-
return Object.assign(
3852-
{
3853-
type: 'header',
3854-
sublayers,
3855-
description: '',
3856-
display_name: '',
3857-
name: '',
3858-
uuid: '',
3859-
},
3860-
{
3861-
description: d.description,
3862-
display_name: d.display_name || basename,
3863-
name: d.name,
3864-
uuid: d.uuid,
3865-
}
3866-
)
3867-
} else if (stac_type === 'collection') {
3868-
const sublayers = []
3869-
const items = stac_data.links.filter((l) =>
3870-
/^item/i.test(l.rel)
3871-
)
3872-
for (let i = 0; i < items.length; i++) {
3873-
const uuid = `${d.uuid}-${i}`
3874-
sublayers.push(
3875-
// we shouldn't need to pre-fetch item data
3876-
Object.assign({}, d, {
3877-
url: items[i].href.replace('./', `${path}/`),
3878-
display_name:
3879-
items[i].title ||
3880-
F_.fileNameFromPath(items[i].href),
3881-
uuid: uuid,
3882-
name: uuid,
3883-
})
3884-
)
3885-
}
3886-
return Object.assign(
3887-
{
3888-
type: 'header',
3889-
sublayers,
3890-
description: '',
3891-
display_name: '',
3892-
name: '',
3893-
uuid: '',
3855+
try {
3856+
console.log(promArr)
3857+
const subls = await Promise.all(promArr)
3858+
sublayers = sublayers.concat(subls)
3859+
} catch (err) {
3860+
console.warn(err)
3861+
resolve(d)
3862+
}
3863+
3864+
resolve(
3865+
Object.assign(
3866+
{
3867+
type: 'header',
3868+
sublayers,
3869+
description: '',
3870+
display_name: '',
3871+
name: '',
3872+
uuid: '',
3873+
},
3874+
{
3875+
description: d.description,
3876+
display_name:
3877+
d.display_name || basename,
3878+
name: d.name,
3879+
uuid: d.uuid,
3880+
}
3881+
)
3882+
)
3883+
} else if (stac_type === 'collection') {
3884+
const sublayers = []
3885+
const items = stac_data.links.filter((l) =>
3886+
/^item/i.test(l.rel)
3887+
)
3888+
for (let i = 0; i < items.length; i++) {
3889+
const uuid = `${d.uuid}-${i}`
3890+
sublayers.push(
3891+
// we shouldn't need to pre-fetch item data
3892+
Object.assign({}, d, {
3893+
url: items[i].href.replace(
3894+
'./',
3895+
`${path}/`
3896+
),
3897+
display_name:
3898+
items[i].title ||
3899+
F_.fileNameFromPath(items[i].href),
3900+
uuid: uuid,
3901+
name: uuid,
3902+
})
3903+
)
3904+
}
3905+
resolve(
3906+
Object.assign(
3907+
{
3908+
type: 'header',
3909+
sublayers,
3910+
description: '',
3911+
display_name: '',
3912+
name: '',
3913+
uuid: '',
3914+
},
3915+
{
3916+
description: d.description,
3917+
display_name:
3918+
d.display_name || basename,
3919+
name: d.name,
3920+
uuid: d.uuid,
3921+
}
3922+
)
3923+
)
3924+
} else if (/^feature(collection)?$/i.test(stac_type)) {
3925+
resolve(
3926+
Object.assign({}, d, {
3927+
display_name: d.display_name || basename,
3928+
})
3929+
)
3930+
} else {
3931+
console.warn('Could not process STAC layer')
3932+
resolve(d)
3933+
}
38943934
},
3895-
{
3896-
description: d.description,
3897-
display_name: d.display_name || basename,
3898-
name: d.name,
3899-
uuid: d.uuid,
3935+
error: (resp) => {
3936+
console.warn(resp)
3937+
resolve(d)
39003938
}
3901-
)
3902-
} else if (/^feature(collection)?$/i.test(stac_type)) {
3903-
return Object.assign({}, d, {
3904-
display_name: d.display_name || basename,
39053939
})
39063940
} else {
3907-
console.warn('Could not process STAC layer')
3908-
return d
3941+
resolve(d)
39093942
}
3910-
}
3911-
return d
3943+
})
39123944
}
39133945
}
39143946

‎src/essence/essence.js‎

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -301,7 +301,7 @@ var essence = {
301301
UserInterface_.updateLayerUpdateButton('DISCONNECTED')
302302
}
303303
},
304-
init: function (config, missionsList, swapping) {
304+
init: async function (config, missionsList, swapping) {
305305
//Save the config data
306306
essence.configData = config
307307

@@ -335,7 +335,7 @@ var essence = {
335335
if (!swapping) urlOnLayers = QueryURL.queryURL()
336336

337337
//Parse all the configData
338-
L_.init(essence.configData, missionsList, urlOnLayers)
338+
await L_.init(essence.configData, missionsList, urlOnLayers)
339339

340340
if (swapping) {
341341
ToolController_.clear()
@@ -440,8 +440,8 @@ var essence = {
440440
'config.json' +
441441
'?nocache=' +
442442
new Date().getTime(),
443-
function (data) {
444-
essence.makeMission(data)
443+
async function (data) {
444+
await essence.makeMission(data)
445445
}
446446
).fail(function () {
447447
console.log(
@@ -455,7 +455,7 @@ var essence = {
455455
})
456456
}
457457
},
458-
makeMission: function (data) {
458+
makeMission: async function (data) {
459459
//Remove swap tool from data.tools
460460
for (var i = data.tools.length - 1; i > 0; i--) {
461461
if (data.tools[i].name === 'Swap') {
@@ -479,7 +479,7 @@ var essence = {
479479
}
480480
}
481481

482-
essence.init(data, L_.missionsList, true)
482+
await essence.init(data, L_.missionsList, true)
483483
},
484484
fina: function () {
485485
if (!essence.finalized) {

0 commit comments

Comments
 (0)
⚡