Skip to content

Commit cd302f7

Browse files
BriannaLindBriannaLind
authored
Update pagination.py
automatically handles spatial and temporal tiling, tracks progress, and has robust error handling. It’s ready to plug into your CLI workflow.
1 parent 081d308 commit cd302f7

File tree

1 file changed

+152
-50
lines changed

1 file changed

+152
-50
lines changed

‎pagination.py‎

Lines changed: 152 additions & 50 deletions
Original file line numberDiff line numberDiff line change
@@ -1,74 +1,176 @@
11
"""
22
pagination.py
3-
Handles Planet API pagination with dynamic spatial/temporal tiling.
4-
Automatically splits large AOIs or long date ranges into smaller requests.
3+
Handles Planet API searches with automatic tiling (spatial & temporal), progress tracking,
4+
and robust error handling. Integrates with filters.py to build dynamic filters.
55
"""
66

7-
import math
8-
import logging
7+
import os
98
import requests
10-
from shapely.geometry import Polygon
11-
from typing import List, Dict, Tuple
9+
import math
10+
import time
11+
from typing import List, Tuple, Dict, Any
12+
from shapely.geometry import Polygon, box
13+
from datetime import datetime, timedelta
1214

13-
logger = logging.getLogger(__name__)
15+
from .filters import build_filters
1416

15-
# Thresholds for auto-tiling
16-
MAX_AOI_DEG2 = 10.0 # Max AOI area in deg² before spatial tiling
17-
MAX_DAYS = 30 # Max days before temporal tiling
17+
# -------------------------
18+
# Configuration thresholds
19+
# -------------------------
20+
SPATIAL_TILE_AREA_THRESHOLD_KM2 = 50000 # e.g., 50,000 km² (~size of Massachusetts)
21+
TEMPORAL_TILE_DAYS_THRESHOLD = 30 # if date range > 30 days, apply temporal tiling
22+
MAX_RETRIES = 5
23+
RETRY_DELAY = 10 # seconds
1824

19-
def estimate_scene_count(aoi_area_deg2: float, days: int, density: float = 1.0) -> int:
25+
# -------------------------
26+
# Spatial tiling
27+
# -------------------------
28+
def tile_aoi(aoi: Polygon, tile_size_deg: float = 1.0) -> List[Polygon]:
2029
"""
21-
Estimate number of scenes for a given AOI and date range.
30+
Split AOI polygon into smaller tiles (default 1° x 1°) for large areas.
31+
2232
Args:
23-
aoi_area_deg2: AOI area in degrees²
24-
days: Number of days in query
25-
density: Scenes per deg² per day (default 1.0)
33+
aoi (Polygon): Input AOI.
34+
tile_size_deg (float): Tile size in degrees (lat/lon).
35+
2636
Returns:
27-
Estimated scene count
37+
List[Polygon]: List of polygon tiles covering AOI.
2838
"""
29-
return math.ceil(aoi_area_deg2 * days * density)
39+
minx, miny, maxx, maxy = aoi.bounds
40+
tiles = []
41+
42+
x_steps = math.ceil((maxx - minx) / tile_size_deg)
43+
y_steps = math.ceil((maxy - miny) / tile_size_deg)
3044

45+
for i in range(x_steps):
46+
for j in range(y_steps):
47+
tile = box(
48+
minx + i * tile_size_deg,
49+
miny + j * tile_size_deg,
50+
min(minx + (i + 1) * tile_size_deg, maxx),
51+
min(miny + (j + 1) * tile_size_deg, maxy)
52+
)
53+
# Only keep tiles that intersect AOI
54+
intersection = tile.intersection(aoi)
55+
if intersection.area > 0:
56+
tiles.append(intersection)
57+
return tiles
3158

32-
def should_tile(aoi_area_deg2: float, days: int) -> Tuple[bool, bool]:
59+
60+
# -------------------------
61+
# Temporal tiling
62+
# -------------------------
63+
def tile_dates(start: datetime, end: datetime, max_days: int = TEMPORAL_TILE_DAYS_THRESHOLD) -> List[Tuple[datetime, datetime]]:
3364
"""
34-
Decide whether spatial or temporal tiling is required.
65+
Split date range into smaller slices if longer than max_days.
66+
3567
Args:
36-
aoi_area_deg2: AOI area in deg²
37-
days: Number of days in query
68+
start (datetime): Start date.
69+
end (datetime): End date.
70+
max_days (int): Maximum number of days per slice.
71+
3872
Returns:
39-
Tuple[bool, bool]: (spatial_tiling, temporal_tiling)
73+
List[Tuple[datetime, datetime]]: List of date tuples.
4074
"""
41-
spatial = aoi_area_deg2 > MAX_AOI_DEG2
42-
temporal = days > MAX_DAYS
43-
return spatial, temporal
75+
total_days = (end - start).days + 1
76+
if total_days <= max_days:
77+
return [(start, end)]
4478

79+
slices = []
80+
current_start = start
81+
while current_start <= end:
82+
current_end = min(current_start + timedelta(days=max_days - 1), end)
83+
slices.append((current_start, current_end))
84+
current_start = current_end + timedelta(days=1)
85+
return slices
4586

46-
def paginate_search(session: requests.Session, search_url: str) -> List[Dict]:
87+
88+
# -------------------------
89+
# Planet API pagination
90+
# -------------------------
91+
def fetch_planet_data(
92+
session: requests.Session,
93+
aois: List[Polygon],
94+
date_ranges: List[Tuple[datetime, datetime]],
95+
max_cloud: float = 0.5,
96+
min_sun_angle: float = 0.0,
97+
item_types: List[str] = ["PSScene4Band"],
98+
page_size: int = 250
99+
) -> Tuple[List[str], List[Dict], List[Dict]]:
47100
"""
48-
Paginate through Planet API search results.
101+
Fetch Planet imagery metadata with automatic tiling.
102+
49103
Args:
50-
session: Authenticated requests.Session
51-
search_url: URL for the first search page
104+
session (requests.Session): Authenticated Planet API session.
105+
aois (List[Polygon]): List of AOIs.
106+
date_ranges (List[Tuple[datetime, datetime]]): List of start/end date ranges.
107+
max_cloud (float): Maximum cloud fraction.
108+
min_sun_angle (float): Minimum sun elevation in degrees.
109+
item_types (List[str]): Planet item types to query.
110+
page_size (int): Results per page.
111+
52112
Returns:
53-
List of results (dicts)
113+
Tuple[List[str], List[Dict], List[Dict]]: ids, geometries, properties
54114
"""
55-
results: List[Dict] = []
56-
next_url = search_url
57-
page_num = 1
58-
59-
while next_url:
60-
try:
61-
resp = session.get(next_url)
62-
resp.raise_for_status()
63-
page = resp.json()
64-
except requests.RequestException as e:
65-
logger.error(f"Request failed at page {page_num}: {e}")
66-
break
67-
68-
features = page.get("features", [])
69-
logger.info(f"Fetched page {page_num} with {len(features)} features")
70-
results.extend(features)
71-
next_url = page.get("_links", {}).get("_next")
72-
page_num += 1
73-
74-
return results
115+
ids: List[str] = []
116+
geometries: List[Dict] = []
117+
properties: List[Dict] = []
118+
119+
total_requests = len(aois) * len(date_ranges)
120+
request_count = 0
121+
122+
for aoi in aois:
123+
# Apply spatial tiling if AOI area is large (> threshold)
124+
# Approximate area in km² (assuming 1° ~ 111 km)
125+
approx_area_km2 = (aoi.bounds[2] - aoi.bounds[0]) * (aoi.bounds[3] - aoi.bounds[1]) * 111 ** 2
126+
if approx_area_km2 > SPATIAL_TILE_AREA_THRESHOLD_KM2:
127+
tiles = tile_aoi(aoi)
128+
else:
129+
tiles = [aoi]
130+
131+
for tile in tiles:
132+
for start, end in tile_dates(date_ranges[0][0], date_ranges[-1][1]):
133+
request_count += 1
134+
print(f"[{request_count}/{total_requests}] Requesting tile {tile.bounds} for {start.date()} to {end.date()}")
135+
136+
filter_json = build_filters(
137+
aois=[tile],
138+
date_ranges=[(start, end)],
139+
max_cloud=max_cloud,
140+
min_sun_angle=min_sun_angle
141+
)
142+
143+
search_request = {
144+
"name": f"tile_{request_count}",
145+
"item_types": item_types,
146+
"filter": filter_json
147+
}
148+
149+
# POST search
150+
try:
151+
response = None
152+
for attempt in range(MAX_RETRIES):
153+
try:
154+
response = session.post("https://api.planet.com/data/v1/searches/", json=search_request)
155+
response.raise_for_status()
156+
break
157+
except requests.RequestException as e:
158+
print(f"Retry {attempt+1}/{MAX_RETRIES} due to {e}")
159+
time.sleep(RETRY_DELAY)
160+
if response is None:
161+
print(f"Failed request for tile {tile.bounds}, skipping.")
162+
continue
163+
164+
search_id = response.json()["id"]
165+
# Fetch paginated results
166+
next_url = f"https://api.planet.com/data/v1/searches/{search_id}/results?_page_size={page_size}"
167+
while next_url:
168+
page = session.get(next_url).json()
169+
ids += [f['id'] for f in page['features']]
170+
geometries += [f['geometry'] for f in page['features']]
171+
properties += [f['properties'] for f in page['features']]
172+
next_url = page["_links"].get("_next")
173+
except Exception as e:
174+
print(f"Error fetching data for tile {tile.bounds}: {e}")
175+
176+
return ids, geometries, properties

0 commit comments

Comments
 (0)
⚡