Skip to content

Commit 542664c

Browse files
committed
kwiver dockerfile, fmv models, fmv ingestion task
1 parent 034c16b commit 542664c

9 files changed

Lines changed: 451 additions & 6 deletions

File tree

dev/kwiver.Dockerfile

Lines changed: 52 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,52 @@
1+
# Start from the KWIVER base image
2+
FROM kitware/kwiver:latest
3+
4+
# Install system packages needed for Python and the app
5+
RUN apt-get update && apt-get install --yes --no-install-recommends \
6+
build-essential wget curl libssl-dev zlib1g-dev libbz2-dev \
7+
libreadline-dev libsqlite3-dev libncurses5-dev libncursesw5-dev \
8+
xz-utils tk-dev git libffi-dev liblzma-dev libpq-dev \
9+
libvips-dev gcc libc6-dev gdal-bin libgdal-dev \
10+
&& rm -rf /var/lib/apt/lists/*
11+
12+
# Install Python 3.10 manually if not available
13+
RUN wget https://www.python.org/ftp/python/3.10.13/Python-3.10.13.tgz && \
14+
tar -xf Python-3.10.13.tgz && \
15+
cd Python-3.10.13 && \
16+
./configure --enable-optimizations && \
17+
make -j"$(nproc)" && \
18+
make altinstall && \
19+
cd .. && rm -rf Python-3.10.13*
20+
21+
RUN python3.10 --version
22+
23+
# Install Python packages
24+
RUN python3.10 -m ensurepip && \
25+
python3.10 -m pip install --upgrade pip
26+
27+
# Install large-image
28+
RUN python3.10 -m pip install large-image[gdal,pil] large-image-converter --find-links https://girder.github.io/large_image_wheels
29+
30+
# Copy your application code
31+
COPY ./setup.py /opt/uvdat-server/setup.py
32+
COPY ./manage.py /opt/uvdat-server/manage.py
33+
COPY ./uvdat /opt/uvdat-server/uvdat
34+
35+
# Install uvdat in editable mode with dev dependencies
36+
RUN python3.10 -m pip install --editable /opt/uvdat-server[dev]
37+
38+
# Copy ffmpeg from static builder
39+
RUN wget -O ffmpeg.tar.xz https://johnvansickle.com/ffmpeg/releases/ffmpeg-release-amd64-static.tar.xz
40+
RUN mkdir /tmp/ffextracted
41+
RUN tar -xvf ffmpeg.tar.xz -C /tmp/ffextracted --strip-components 1
42+
43+
# Copy ffmpeg into final image
44+
RUN cp /tmp/ffextracted/ffmpeg /usr/local/bin/
45+
RUN cp /tmp/ffextracted/ffprobe /usr/local/bin/
46+
47+
# Setup environment
48+
49+
50+
# Set working directory
51+
WORKDIR /opt/uvdat-server
52+

docker-compose.override.yml

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,8 @@ services:
22
django:
33
build:
44
context: .
5-
dockerfile: ./dev/Dockerfile
5+
dockerfile: ./dev/kwiver.Dockerfile
6+
entrypoint: ["python3.10"]
67
command: [ "./manage.py", "runserver", "0.0.0.0:8000" ]
78
# Log printing via Rich is enhanced by a TTY
89
tty: true
@@ -15,12 +16,12 @@ services:
1516
- postgres
1617
- rabbitmq
1718
- minio
18-
platform: linux/amd64
1919

2020
celery:
2121
build:
2222
context: .
23-
dockerfile: ./dev/Dockerfile
23+
dockerfile: ./dev/kwiver.Dockerfile
24+
entrypoint: ["python3.10" , "-m"]
2425
command:
2526
[
2627
"celery",

sample_data/fmv.json

Lines changed: 28 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,28 @@
1+
[{
2+
"type": "Context",
3+
"name": "FMV",
4+
"default_map_center": [
5+
34.8019,
6+
-86.1794
7+
],
8+
"default_map_zoom": 6,
9+
"datasets": [
10+
{
11+
"name": "FMV",
12+
"description": "FMV Testing",
13+
"category": "fmv",
14+
"metadata": {},
15+
"files": [
16+
{
17+
"path": "./data/FMV/fmv.mpg",
18+
"url": "https://data.kitware.com/api/v1/file/604a5a532fa25629b931c673/download",
19+
"name": "FMV Test Video",
20+
"type": "fmv",
21+
"action": "replace",
22+
"metadata": {
23+
}
24+
}
25+
]
26+
}
27+
]
28+
}]

uvdat/core/admin.py

Lines changed: 21 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -23,6 +23,8 @@
2323
VectorFeatureRowData,
2424
VectorFeatureTableData,
2525
VectorMapLayer,
26+
FMVLayer,
27+
FMVVectorFeature
2628
)
2729

2830

@@ -73,6 +75,23 @@ def get_map_layer_index(self, obj):
7375
return obj.map_layer.index
7476

7577

78+
class FMVLayerAdmin(admin.ModelAdmin):
79+
list_display = ['id', 'name', 'dataset', 'get_dataset_name', 'index', 'geojson_file', 'fmv_video', 'bounds']
80+
81+
def get_dataset_name(self, obj):
82+
return obj.dataset.name
83+
84+
85+
class FMVVectorFeatureAdmin(admin.ModelAdmin):
86+
list_display = ['id', 'get_dataset_name', 'get_map_layer_index']
87+
88+
def get_dataset_name(self, obj):
89+
return obj.map_layer.dataset.name
90+
91+
def get_map_layer_index(self, obj):
92+
return obj.map_layer.index
93+
94+
7695
class SourceRegionAdmin(admin.ModelAdmin):
7796
list_display = ['id', 'name', 'get_dataset_name']
7897

@@ -229,6 +248,8 @@ class DisplayConfigurationAdmin(admin.ModelAdmin):
229248
admin.site.register(RasterMapLayer, RasterMapLayerAdmin)
230249
admin.site.register(VectorMapLayer, VectorMapLayerAdmin)
231250
admin.site.register(VectorFeature, VectorFeatureAdmin)
251+
admin.site.register(FMVLayer, FMVLayerAdmin)
252+
admin.site.register(FMVVectorFeature, FMVVectorFeatureAdmin)
232253
admin.site.register(SourceRegion, SourceRegionAdmin)
233254
admin.site.register(DerivedRegion, DerivedRegionAdmin)
234255
admin.site.register(Network, NetworkAdmin)
Lines changed: 46 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,46 @@
1+
# Generated by Django 5.0.7 on 2025-06-06 19:10
2+
3+
import django.contrib.gis.db.models.fields
4+
import django.db.models.deletion
5+
import django_extensions.db.fields
6+
import s3_file_field.fields
7+
from django.db import migrations, models
8+
9+
10+
class Migration(migrations.Migration):
11+
12+
dependencies = [
13+
('core', '0004_displayconfiguration'),
14+
]
15+
16+
operations = [
17+
migrations.CreateModel(
18+
name='FMVLayer',
19+
fields=[
20+
('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
21+
('created', django_extensions.db.fields.CreationDateTimeField(auto_now_add=True, verbose_name='created')),
22+
('modified', django_extensions.db.fields.ModificationDateTimeField(auto_now=True, verbose_name='modified')),
23+
('metadata', models.JSONField(blank=True, null=True)),
24+
('default_style', models.JSONField(blank=True, null=True)),
25+
('index', models.IntegerField(null=True)),
26+
('name', models.CharField(blank=True, max_length=255)),
27+
('bounds', django.contrib.gis.db.models.fields.PolygonField(blank=True, help_text='Bounds/Extents of the Layer', null=True, srid=4326)),
28+
('fmv_source_video', s3_file_field.fields.S3FileField(null=True)),
29+
('fmv_video', s3_file_field.fields.S3FileField(null=True)),
30+
('geojson_file', s3_file_field.fields.S3FileField(null=True)),
31+
('dataset', models.ForeignKey(null=True, on_delete=django.db.models.deletion.CASCADE, to='core.dataset')),
32+
],
33+
options={
34+
'abstract': False,
35+
},
36+
),
37+
migrations.CreateModel(
38+
name='FMVVectorFeature',
39+
fields=[
40+
('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
41+
('geometry', django.contrib.gis.db.models.fields.GeometryField(srid=4326)),
42+
('properties', models.JSONField()),
43+
('map_layer', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='core.fmvlayer')),
44+
],
45+
),
46+
]

uvdat/core/models/__init__.py

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,7 @@
55
from .file_item import FileItem
66
from .layer_collection import LayerCollection
77
from .layer_representation import LayerRepresentation
8-
from .map_layers import AbstractMapLayer, RasterMapLayer, VectorFeature, VectorMapLayer
8+
from .map_layers import AbstractMapLayer, RasterMapLayer, VectorFeature, VectorMapLayer, FMVLayer, FMVVectorFeature
99
from .netcdf import NetCDFData, NetCDFImage, NetCDFLayer
1010
from .networks import Network, NetworkEdge, NetworkNode
1111
from .processing_task import ProcessingTask
@@ -37,4 +37,6 @@
3737
VectorFeatureTableData,
3838
VectorFeatureRowData,
3939
DisplayConfiguration,
40+
FMVVectorFeature,
41+
FMVLayer
4042
]

uvdat/core/models/map_layers.py

Lines changed: 61 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -39,7 +39,7 @@ def set_bounds(self):
3939
self.bounds = Polygon.from_bbox(
4040
(bbox['xmin'], bbox['ymin'], bbox['xmax'], bbox['ymax'])
4141
)
42-
elif isinstance(self, VectorMapLayer):
42+
elif isinstance(self, (VectorMapLayer, FMVLayer)):
4343
geojson_data = self.read_geojson_data()
4444
if 'features' in geojson_data:
4545
geometries = [shape(feature['geometry']) for feature in geojson_data['features']]
@@ -104,6 +104,66 @@ def read_geojson_data(self) -> dict:
104104
return json.load(self.geojson_file.open())
105105

106106

107+
class FMVLayer(AbstractMapLayer):
108+
fmv_source_video = S3FileField(null=True)
109+
fmv_video = S3FileField(null=True)
110+
geojson_file = S3FileField(null=True)
111+
112+
def write_geojson_data(self, content: str | dict):
113+
if isinstance(content, str):
114+
data = content
115+
elif isinstance(content, dict):
116+
data = json.dumps(content)
117+
else:
118+
raise Exception(f'Invalid content type supplied: {type(content)}')
119+
120+
self.geojson_file.save('vectordata.geojson', ContentFile(data.encode()))
121+
122+
def read_geojson_data(self) -> dict:
123+
"""Read and load the data from geojson_file into a dict."""
124+
return json.load(self.geojson_file.open())
125+
126+
def get_ground_frame_mapping(self) -> dict:
127+
"""Return a mapping from frameId -> 4-point polygon coordinates."""
128+
result = {}
129+
features = FMVVectorFeature.objects.filter(
130+
map_layer=self,
131+
properties__type='ground_frame',
132+
).exclude(properties__frameId__isnull=True)
133+
134+
for feature in features:
135+
frame_id = feature.properties.get("frameId")
136+
geom = feature.geometry
137+
138+
if geom.geom_type == "Polygon":
139+
coords = list(geom.exterior.coords)
140+
if len(coords) >= 4:
141+
# Return first 4 unique corners, not repeating the closing point
142+
result[frame_id] = coords[:4]
143+
elif geom.geom_type == "MultiPolygon":
144+
# Pick the largest polygon by area, then return its first 4 corners
145+
largest = max(geom, key=lambda p: p.area)
146+
coords = list(largest.exterior.coords)
147+
if len(coords) >= 4:
148+
result[frame_id] = coords[:4]
149+
else:
150+
# Optional: log or skip unexpected geometries
151+
continue
152+
153+
return result
154+
155+
156+
@receiver(models.signals.pre_delete, sender=FMVLayer)
157+
def delete__fmvvectorcontent(sender, instance, **kwargs):
158+
if instance.geojson_file:
159+
instance.geojson_file.delete(save=False)
160+
161+
class FMVVectorFeature(models.Model):
162+
map_layer = models.ForeignKey(FMVLayer, on_delete=models.CASCADE)
163+
geometry = geomodels.GeometryField()
164+
properties = models.JSONField()
165+
166+
107167
@receiver(models.signals.pre_delete, sender=VectorMapLayer)
108168
def delete__vectorcontent(sender, instance, **kwargs):
109169
if instance.geojson_file:

uvdat/core/tasks/dataset.py

Lines changed: 21 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -14,13 +14,27 @@
1414
process_geopackage,
1515
process_tabular_vector_feature_data,
1616
)
17+
from .fmv import create_fmv_layer
1718
from .netcdf import create_netcdf_data_layer
1819
from .networks import create_network
1920
from .regions import create_source_regions
2021

2122
logger = logging.getLogger(__name__)
2223

2324

25+
valid_video_format = (
26+
"mp4",
27+
"webm",
28+
"avi",
29+
"mov",
30+
"wmv",
31+
"mpg",
32+
"mpeg",
33+
"mp2",
34+
"ogg",
35+
"flv",
36+
)
37+
2438
@shared_task
2539
def convert_dataset(
2640
dataset_id,
@@ -79,7 +93,8 @@ def convert_dataset(
7993
vector_map_layer.pk, tabular_geojson, tabular_matcher
8094
)
8195
vector_map_layer.set_bounds()
82-
96+
elif file_name.endswith(valid_video_format):
97+
create_fmv_layer(file_to_convert, style_options, file_name, file_metadata)
8398
elif file_name.endswith(('.tif', '.tiff')):
8499
# Handle Raster files
85100
raster_map_layer = create_raster_map_layer(
@@ -117,6 +132,7 @@ def process_file_item(self, file_item_id):
117132
raster_map_layers = []
118133
vector_map_layers = []
119134
netcdf_map_layers = []
135+
fmv_map_layers = []
120136
processing_task.update(status=ProcessingTask.Status.RUNNING)
121137
try:
122138
if file_name.endswith('.gpkg'):
@@ -141,6 +157,9 @@ def process_file_item(self, file_item_id):
141157
vector_map_layer.set_bounds()
142158
vector_map_layers.append(vector_map_layer)
143159

160+
elif file_name.endswith(valid_video_format):
161+
fmv_map_layer = create_fmv_layer(file_item, style_options, file_name, file_metadata)
162+
fmv_map_layers.append(fmv_map_layer)
144163
elif file_name.endswith(('.tif', '.tiff')):
145164
# Handle Raster files
146165
raster_map_layer = create_raster_map_layer(
@@ -176,6 +195,7 @@ def process_file_item(self, file_item_id):
176195
'raster_map_layers': [rml.id for rml in raster_map_layers],
177196
'vector_map_layers': [vml.id for vml in vector_map_layers],
178197
'net_cdf_map_layers': netcdf_map_layers,
198+
'fmv_map_layers': [fmv.id for fmv in fmv_map_layers]
179199
}
180200
},
181201
)

0 commit comments

Comments
 (0)