geowatch.tasks.fusion.datamodules.spacetime_grid_builder module

CommandLine

# Benchmark time sampling
LINE_PROFILE=1 xdoctest -m geowatch.tasks.fusion.datamodules.spacetime_grid_builder __doc__:0

Todo

• [ ] The functions that take too many arguments should be refactored as

  object oriented code and use object attribute to store the extra data.

Example

>>> # xdoctest: +REQUIRES(env:LINE_PROFILE)
>>> from geowatch.tasks.fusion.datamodules.spacetime_grid_builder import *  # NOQA
>>> import ndsampler
>>> import kwcoco
>>> dset = kwcoco.CocoDataset.demo('vidshapes2', num_frames=100)
>>> window_overlap = 0.0
>>> time_dims = 1
>>> window_dims = (32, 32)
>>> keepbound = False
>>> time_sampling = 'soft2+distribute'
>>> sample_grid1 = SpacetimeGridBuilder(
>>>     dset, time_dims, window_dims, window_overlap, use_cache=0,
>>>     use_centered_positives=0,
>>>     use_annot_info=0,
>>>     time_sampling='contiguous').build()
>>> boxes = [kwimage.Boxes.from_slice(target['space_slice'], clip=False).to_xywh() for target in sample_grid1['targets']]
>>> all_boxes = kwimage.Boxes.concatenate(boxes)
>>> assert np.all(all_boxes.height == window_dims[0])
>>> assert np.all(all_boxes.width == window_dims[1])

Example

>>> # xdoctest: +REQUIRES(env:SMART_DATA_DVC_DPATH)
>>> from geowatch.tasks.fusion.datamodules.spacetime_grid_builder import *  # NOQA
>>> import geowatch
>>> import kwcoco
>>> dvc_dpath = geowatch.find_dvc_dpath(tags='phase3_data', hardware='ssd')
>>> coco_fpath = dvc_dpath / 'Drop8-ARA-Median10GSD-V1/KR_R002/imganns-KR_R002-rawbands.kwcoco.zip'
>>> coco_dset = kwcoco.CocoDataset(coco_fpath)
>>> window_dims = 128
>>> time_dims = 5
>>> builder = SpacetimeGridBuilder(
>>>     coco_dset,
>>>     time_dims,
>>>     window_dims,
>>>     time_sampling='soft2+distribute',
>>>     time_kernel='-1y,-8m,-2w,0,2w,8m,1y',
>>>     keepbound=True,
>>>     use_annot_info=1,
>>>     use_grid_positives=1,
>>>     use_centered_positives=True,
>>>     respect_valid_regions=False,  # enabling this is slow
>>>     use_cache=0
>>> )
>>> grid = builder.build()
>>> # xdoctest: +REQUIRES(--show)
>>> import kwplot
>>> plt = kwplot.autoplt()
>>> canvas = builder.visualize(max_vids=5, max_frames=5)
>>> kwplot.imshow(canvas, doclf=1, fnum=2)
>>> plt.gca().set_title(ub.codeblock(
    '''
    Sampled using larger scaled windows
    '''))
>>> kwplot.show_if_requested()

Example

>>> # xdoctest: +REQUIRES(env:SMART_DATA_DVC_DPATH)
>>> from geowatch.tasks.fusion.datamodules.spacetime_grid_builder import *  # NOQA
>>> import geowatch
>>> import kwcoco
>>> dvc_dpath = geowatch.find_dvc_dpath(tags='phase3_data', hardware='hdd')
>>> #coco_fpath = dvc_dpath / 'submodules/Drop8-ARA-Cropped2GSD-V1/KR_R002/imganns-KR_R002-rawbands.kwcoco.zip'
>>> coco_fpath = dvc_dpath / 'submodules/Drop8-ARA-Cropped2GSD-V1/AE_R001/imganns-AE_R001-rawbands.kwcoco.zip'
>>> coco_dset = kwcoco.CocoDataset(coco_fpath)
>>> wh_lut = coco_dset.videos().lookup(['width', 'height'])
>>> dsizes = list(zip(wh_lut['width'], wh_lut['height']))
>>> window_dims = 128
>>> builder = SpacetimeGridBuilder(
>>>     dset=coco_dset,
>>>     window_dims=window_dims,
>>>     time_sampling='soft2+distribute',
>>>     time_kernel='-1y,-8m,-2w,0,2w,8m,1y',
>>>     dynamic_fixed_resolution=None,
>>>     #dynamic_fixed_resolution={'max_winspace_full_dims': [1024, 1024]},
>>>     keepbound=True,
>>>     use_annot_info=1,
>>>     use_grid_positives=1,
>>>     use_centered_positives=True,
>>>     respect_valid_regions=False,  # enabling this is slow
>>>     use_cache=1
>>> )
>>> grid = builder.build()
>>> # xdoctest: +REQUIRES(--show)
>>> import kwplot
>>> plt = kwplot.autoplt()
>>> #infos = [ub.udict(v) & {'video_name', 'vidspace_full_dims'} for v in grid['vidid_to_meta'].values()]
>>> #infos = sorted([d | {'area': np.prod(d['vidspace_full_dims'])} for d in infos], key=lambda d: d['area'])
>>> # Show big and small clusters
>>> #chosen = [infos[30], infos[-1]]
>>> video_ids = None
>>> video_ids = coco_dset.videos(names=['AE_R001_CLUSTER_073', 'AE_R001_CLUSTER_000']).ids
>>> canvas = builder.visualize(max_vids=10, max_frames=1, video_ids=video_ids)
>>> print(f'canvas.shape={canvas.shape}')
>>> kwimage.imwrite('canvas.jpg', canvas)
>>> kwplot.imshow(canvas, doclf=1, fnum=2)
>>> relevant_params = ub.udict(builder.kw) & {'window_dims', 'window_overlap', 'time_kernel', 'use_grid_positives', 'use_centered_positives', 'dynamic_fixed_resolution'}
>>> relevant_text = ub.urepr(relevant_params, concise=1, nobr=1, si=1, nl=0)
>>> print(f'relevant_text = {ub.urepr(relevant_text, nl=1)}')
>>> plt.gca().set_title(ub.codeblock(
    f'''
    {relevant_text}

Places a red dot where there is a negative sample (at the center of the negative window)

Places a blue dot where there is a positive sample

Draws a yellow polygon over invalid spatial regions. ‘’’))

>>> kwplot.show_if_requested()

class geowatch.tasks.fusion.datamodules.spacetime_grid_builder.SpacetimeGridBuilder(dset, time_dims=None, window_dims=None, window_overlap=0.0, negative_classes=None, keepbound=True, include_sensors=None, exclude_sensors=None, select_images=None, select_videos=None, time_sampling='hard+distribute', time_span=None, time_kernel=None, use_annot_info=True, use_grid_positives=True, use_grid_negatives=True, use_centered_positives=True, window_space_scale=None, set_cover_algo=None, respect_valid_regions=True, dynamic_fixed_resolution=None, workers=0, use_cache=1)

Bases: object

A helper class to help build a grid of spacetime windows for a coco dataset.

The basic idea is that you will slide a spacetime window over the dataset and mark where positive andnegative “windows” are. We also put windows directly on positive annotations if desired.

See the visualize_sample_grid() for a visualization of what the sample grid looks like.

Example

>>> # xdoctest: +REQUIRES(env:DVC_DPATH)
>>> import os
>>> from geowatch.tasks.fusion.datamodules.spacetime_grid_builder import *  # NOQA
>>> import geowatch
>>> dvc_dpath = geowatch.find_dvc_dpath(tags='phase2_data', hardware='ssd')
>>> coco_fpath = dvc_dpath / 'Drop6/data_train_split1.kwcoco.zip'
>>> dset = geowatch.coerce_kwcoco(coco_fpath)
>>> window_overlap = 0.0
>>> window_dims = (128, 128)
>>> time_dims = 2
>>> sample_grid = SpacetimeGridBuilder(dset, time_dims, window_dims).build()
>>> time_sampling = 'hard+distribute'
>>> positives = list(ub.take(sample_grid['targets'], sample_grid['positives_indexes']))

Parameters:

• dset (kwcoco.CocoDataset) – coco dataset

• time_dims (int) – number of time steps (Deprecated, use time_kernel)

• window_dims (Tuple[int, int] | str) – spatial height, width of the sample region or a string code.

• window_overlap (float) – fractional spatial overlap

• set_cover_algo (str | None) – Algorithm used to find set cover of image IDs. Options are ‘approx’ (a greedy solution) or ‘exact’ (an ILP solution). If None is passed, set cover is not computed. The ‘exact’ method requires the packe pulp, available at PyPi.

• window_space_scale (str) – Code indicating the scale at which to sample. If None uses the videospace GSD.

• use_grid_positives (bool) – if False, will remove any grid sample that contains a positive example. In this case use_centered_positives should be True.

• use_grid_negatives (bool | str) – Use non-annotation locations as negatives. Can be “cleared”.

• use_centered_positives (bool) – extend the grid with extra off-axis samples where positive annotations are centered. TODO: we could do a box packing to reduce the potential size here.

• use_annot_info (bool) – if True allows using annotation information to get a better train-time grid. Should not be used at test-time.

• time_span (str) – indicates the desired start/stop date range of the sample (Deprecated, use time_kernel)

• time_kernel (str) – mutually exclusive with time span.

• time_sampling (str) – code for specific temporal sampler: see temporal_sampling/sampler.py for more information.

• exclude_sensors (List[str]) – A list of sensors to exclude from the grid

• negative_classes (List[str]) – indicate class names that should not count towards a region being marked as positive. NOTE: This is old and deprecated.

• respect_valid_regions (bool) – if True, only place windows in valid regions

• dynamic_fixed_resolution (None | Any) – Experimental. Added in 0.17.1 example value: {‘max_winspace_full_dims’: [1000, 1000]}

• workers (int) – parallel workers

• use_cache (bool) – uses a disk cache if True

build()

This is the main driver that builds the sample grid.

Example

>>> # xdoctest: +REQUIRES(env:DVC_DPATH)
>>> import os
>>> from geowatch.tasks.fusion.datamodules.spacetime_grid_builder import *  # NOQA
>>> import geowatch
>>> dvc_dpath = geowatch.find_dvc_dpath(tags='phase2_data', hardware='ssd')
>>> coco_fpath = dvc_dpath / 'Aligned-Drop3-TA1-2022-03-10/combo_LM_nowv_vali.kwcoco.json'
>>> dset = kwcoco.CocoDataset(coco_fpath)
>>> window_overlap = 0.5
>>> time_dims = 2
>>> window_dims = (128, 128)
>>> keepbound = False
>>> exclude_sensors = None
>>> set_cover_algo = 'approx'
>>> sample_grid = SpacetimeGridBuilder(dset, time_dims, window_dims, window_overlap, set_cover_algo=set_cover_algo).build()
>>> time_sampling = 'hard+distribute'
>>> positives = list(ub.take(sample_grid['targets'], sample_grid['positives_indexes']))

Example

>>> from geowatch.tasks.fusion.datamodules.spacetime_grid_builder import *  # NOQA
>>> import ndsampler
>>> import kwcoco
>>> dset = kwcoco.CocoDataset.demo('vidshapes2-multispectral', num_frames=30)
>>> window_overlap = 0.0
>>> time_dims = 3
>>> window_dims = (256, 256)
>>> keepbound = False
>>> time_sampling = 'soft2+distribute'
>>> sample_grid1 = SpacetimeGridBuilder(
>>>     dset, time_dims, window_dims, window_overlap,
>>>     time_sampling='soft2+distribute').build()
>>> boxes = [kwimage.Boxes.from_slice(target['space_slice'], clip=False).to_xywh() for target in sample_grid1['targets']]
>>> all_boxes = kwimage.Boxes.concatenate(boxes)
>>> assert np.all(all_boxes.height == window_dims[0])
>>> assert np.all(all_boxes.width == window_dims[1])

Example

>>> from geowatch.tasks.fusion.datamodules.spacetime_grid_builder import *  # NOQA
>>> import ndsampler
>>> import kwcoco
>>> dset = kwcoco.CocoDataset.demo('vidshapes2-multispectral', num_frames=30)
>>> window_overlap = 0.0
>>> time_dims = 3
>>> window_dims = (32, 32)
>>> keepbound = False
>>> time_sampling = 'soft2+distribute'
>>> sample_grid1 = SpacetimeGridBuilder(
>>>     dset, time_dims, window_dims, window_overlap, exclude_sensors='Foo',
>>>     time_sampling='soft2+distribute').build()
>>> sample_grid2 = SpacetimeGridBuilder(
>>>     dset, time_dims, window_dims, window_overlap,
>>>     time_sampling='contiguous+pairwise').build()

ub.peek(sample_grid1[‘vidid_to_time_sampler’].values()).show_summary(fnum=1) ub.peek(sample_grid2[‘vidid_to_time_sampler’].values()).show_summary(fnum=2) _ = xdev.profile_now(sample_video_spacetime_targets)(dset, time_dims, window_dims, window_overlap)

import xdev globals().update(xdev.get_func_kwargs(sample_video_spacetime_targets))

visualize(max_vids=2, max_frames=6, video_ids=None)

Debug visualization for sampling grid

Draws multiple frames.

Places a red dot where there is a negative sample (at the center of the negative window)

Places a blue dot where there is a positive sample

Draws a yellow polygon over invalid spatial regions.

Todo

• [x] Make this the SpacetimeGridBuilder.visualize method

Notes

• Dots are more intense when there are more temporal coverage of that dot.

• Dots are placed on the center of the window. They do not indicate its extent.

• Dots are blue if they overlap any annotation in their temporal region so they may visually be near an annotation.

Parameters:

• max_vids (int) – maximum videos that can be shown

• max_frames (int) – maximum frame from each video to draw

• video_ids (List[int] | None) – if specified, show these videos

Example

>>> from geowatch.tasks.fusion.datamodules.spacetime_grid_builder import *  # NOQA
>>> from geowatch.demo.smart_kwcoco_demodata import demo_kwcoco_multisensor
>>> dset = coco_dset = demo_kwcoco_multisensor(num_frames=3, dates=True, geodata=True, heatmap=True, rng=10)
>>> window_overlap = 0.2
>>> window_dims = (64, 64)
>>> time_sampling = 'soft2+distribute'
>>> use_centered_positives = True
>>> use_grid_positives = 1
>>> time_dims = 3
>>> time_kernel = '-1d,0d,1d'
>>> builder = SpacetimeGridBuilder(
>>>     dset=dset, time_dims=time_dims, time_kernel=time_kernel, window_dims=window_dims,
>>>     window_overlap=window_overlap, time_sampling=time_sampling,
>>>     use_grid_positives=use_grid_positives,
>>>     use_centered_positives=use_centered_positives)
>>> grid = builder.build()
>>> # xdoctest: +REQUIRES(--show)
>>> import kwplot
>>> plt = kwplot.autoplt()
>>> canvas = builder.visualize(max_vids=1, max_frames=3)
>>> kwplot.imshow(canvas, doclf=1)
>>> relevant_params = ub.udict(builder.kw) & {'window_dims', 'window_overlap', 'time_dims', 'use_grid_positives', 'use_centered_positives'}
>>> relevant_text = ub.urepr(relevant_params, concise=1, nobr=1, si=1, nl=0)
>>> print(f'relevant_text = {ub.urepr(relevant_text, nl=1)}')
>>> plt.gca().set_title(ub.codeblock(
    f'''
    {relevant_text}

Places a red dot where there is a negative sample (at the center of the negative window)

Places a blue dot where there is a positive sample

Draws a yellow polygon over invalid spatial regions. ‘’’))

>>> kwplot.show_if_requested()
>>> #
>>> # Now demo this same grid, but where we are sampling at a different resolution
>>> window_space_scale = 0.3
>>> builder2 = SpacetimeGridBuilder(
>>>     dset=dset, time_dims=time_dims, window_dims=window_dims, window_overlap=window_overlap,
>>>     time_sampling=time_sampling, use_grid_positives=use_grid_positives,
>>>     use_centered_positives=use_centered_positives,
>>>     window_space_scale=window_space_scale)
>>> sample_grid2 = builder2.build()
>>> # xdoctest: +REQUIRES(--show)
>>> import kwplot
>>> plt = kwplot.autoplt()
>>> canvas = builder2.visualize(max_vids=1, max_frames=3)
>>> kwplot.imshow(canvas, doclf=1, fnum=2)
>>> plt.gca().set_title(ub.codeblock(
    '''
    Sampled using larger scaled windows
    '''))
>>> kwplot.show_if_requested()

Example

>>> # xdoctest: +REQUIRES(env:DVC_DPATH)
>>> from geowatch.tasks.fusion.datamodules.spacetime_grid_builder import *  # NOQA
>>> import geowatch
>>> # dset = coco_dset = demo_kwcoco_multisensor(dates=True, geodata=True, heatmap=True)
>>> dvc_dpath = geowatch.find_dvc_dpath(hardware='ssd', tags='phase2_data')
>>> #coco_fpath = dvc_dpath / 'Drop2-Aligned-TA1-2022-02-15/combo_DILM_train.kwcoco.json'
>>> coco_fpath = dvc_dpath / 'Drop6/data_vali_split1.kwcoco.zip'
>>> big_dset = kwcoco.CocoDataset(coco_fpath)
>>> vid_gids = big_dset.videos(names=['KR_R002']).images.lookup('id')[0]
>>> idxs = np.linspace(0, len(vid_gids) - 1, 12).round().astype(int)
>>> vid_gids = list(ub.take(vid_gids, idxs))
>>> dset = big_dset.subset(vid_gids)
>>> window_overlap = 0.0
>>> window_dims = (3, 256, 256)
>>> keepbound = False
>>> time_sampling = 'soft2+distribute'
>>> use_centered_positives = 0
>>> use_grid_positives = 1
>>> window_space_scale = '30GSD'
>>> sample_grid = sample_video_spacetime_targets(
>>>     dset, window_dims, window_overlap, time_sampling=time_sampling,
>>>     use_grid_positives=True,
>>>     use_centered_positives=use_centered_positives,
>>>     window_space_scale=window_space_scale)
>>> print(list(ub.unique([t['space_slice'] for t in sample_grid['targets']], key=ub.hash_data)))
>>> # xdoctest: +REQUIRES(--show)
>>> import kwplot
>>> plt = kwplot.autoplt()
>>> canvas = _visualize_sample_grid(dset, sample_grid, max_vids=1, max_frames=12)
>>> kwplot.imshow(canvas, doclf=1)
>>> kwplot.show_if_requested()
>>> plt.gca().set_title(ub.codeblock(
>>>     f'''
>>>     Sample window {window_dims} @ {window_space_scale}
>>>     '''))

class geowatch.tasks.fusion.datamodules.spacetime_grid_builder.ImagePropertyCacher(dset)

Bases: object

Helper class for caching image property lookups

get_warp_vid_from_img()

Abstract the transform to bring us into whatever the internal “window space” is. Depends on whatever “window_scale” is computed as.

get_image_valid_region_in_vidspace()

geowatch.tasks.fusion.datamodules.spacetime_grid_builder.sample_video_spacetime_targets(dset, time_dims=None, window_dims=None, window_overlap=0.0, negative_classes=None, keepbound=True, include_sensors=None, exclude_sensors=None, select_images=None, select_videos=None, time_sampling='hard+distribute', time_span='2y', time_kernel=None, use_annot_info=True, use_grid_positives=True, use_grid_negatives=True, use_centered_positives=True, window_space_scale=None, set_cover_algo=None, respect_valid_regions=True, dynamic_fixed_resolution=None, workers=0, use_cache=1)

Old API. Deprecated.