geowatch.tasks.fusion.datamodules.temporal_sampling.sampler module

This module defines our generalized time sampling strategy.

This is used to define our dilated time sampling.

This following doctest illustrates the method on project data.

CommandLine

SMART_DATA_DVC_DPATH=1 XDEV_PROFILE=1 xdoctest -m geowatch.tasks.fusion.datamodules.temporal_sampling __doc__:3
SMART_DATA_DVC_DPATH=1 xdoctest -m geowatch.tasks.fusion.datamodules.temporal_sampling __doc__:3

xdoctest -m geowatch.tasks.fusion.datamodules.temporal_sampling __doc__:0 --show
xdoctest -m geowatch.tasks.fusion.datamodules.temporal_sampling __doc__:1 --show

Example

>>> # Basic overview demo of the algorithm
>>> from geowatch.tasks.fusion.datamodules.temporal_sampling.sampler import *  # NOQA
>>> import geowatch
>>> dset = geowatch.coerce_kwcoco('geowatch-msi', geodata=True, dates=True, num_frames=16, image_size=(8, 8))
>>> vidid = dset.dataset['videos'][0]['id']
>>> self = TimeWindowSampler.from_coco_video(
>>>     dset, vidid,
>>>     time_window=5,
>>>     affinity_type='soft2', time_span='8m', update_rule='distribute',
>>> )
>>> # xdoctest: +REQUIRES(--show)
>>> import kwplot
>>> plt = kwplot.autoplt()
>>> self.show_summary(samples_per_frame=3, fnum=3)
>>> self.show_procedure(fnum=1)
>>> plt.subplots_adjust(top=0.9)
>>> kwplot.show_if_requested()

Example

>>> # Demo multiple different settings
>>> from geowatch.tasks.fusion.datamodules.temporal_sampling.sampler import *  # NOQA
>>> import geowatch
>>> dset = geowatch.coerce_kwcoco('geowatch-msi', geodata=True, dates=True, num_frames=16, image_size=(8, 8), num_videos=1)
>>> vidid = dset.dataset['videos'][0]['id']
>>> self = TimeWindowSampler.from_coco_video(
>>>     dset, vidid,
>>>     time_window=7,
>>>     affinity_type='uniform', time_span='8m', update_rule='',
>>> )
>>> # xdoctest: +REQUIRES(--show)
>>> import kwplot
>>> plt = kwplot.autoplt()
>>> self.update_affinity(affinity_type='contiguous')
>>> self.show_summary(samples_per_frame=3, fnum=1)
>>> self.show_procedure(fnum=4)
>>> plt.subplots_adjust(top=0.9)
>>> self.update_affinity(affinity_type='soft2')
>>> self.show_summary(samples_per_frame=3, fnum=2)
>>> self.show_procedure(fnum=5)
>>> plt.subplots_adjust(top=0.9)
>>> self.update_affinity(affinity_type='hardish3')
>>> self.show_summary(samples_per_frame=3, fnum=3)
>>> self.show_procedure(fnum=6)
>>> self.update_affinity(affinity_type='uniform')
>>> self.show_summary(samples_per_frame=3, fnum=3)
>>> self.show_procedure(fnum=6)
>>> plt.subplots_adjust(top=0.9)
>>> kwplot.show_if_requested()

Example

>>> # Demo corner case where there are too few observations
>>> from geowatch.tasks.fusion.datamodules.temporal_sampling.sampler import *  # NOQA
>>> import geowatch
>>> dset = geowatch.coerce_kwcoco('geowatch-msi', geodata=True, dates=True, num_frames=1, num_videos=1, image_size=(8, 8))
>>> vidid = dset.dataset['videos'][0]['id']
>>> self = TimeWindowSampler.from_coco_video(
>>>     dset, vidid,
>>>     time_window=2,
>>>     affinity_type='hardish3', time_span='1y', update_rule='',
>>> )
>>> idxs = self.sample()
>>> assert idxs == [0, 0]
>>> # xdoctest: +REQUIRES(--show)
>>> import kwplot
>>> plt = kwplot.autoplt()
>>> self.show_summary(samples_per_frame=3, fnum=1)
>>> self.show_procedure(fnum=4)
>>> plt.subplots_adjust(top=0.9)
>>> kwplot.show_if_requested()

Example

>>> # xdoctest: +REQUIRES(env:SMART_DATA_DVC_DPATH)
>>> from geowatch.tasks.fusion.datamodules.temporal_sampling.sampler import *  # NOQA
>>> import geowatch
>>> data_dvc_dpath = geowatch.find_dvc_dpath(tags='phase2_data', hardware='auto')
>>> coco_fpath = data_dvc_dpath / 'Drop6/imganns-KR_R001.kwcoco.zip'
>>> dset = geowatch.coerce_kwcoco(coco_fpath)
>>> vidid = dset.dataset['videos'][0]['id']
>>> self = TimeWindowSampler.from_coco_video(
>>>     dset, vidid,
>>>     time_kernel='-1y,-8m,-2w,0,2w,8m,1y',
>>>     affinity_type='soft4', update_rule='', deterministic=True
>>>     #time_window=5,
>>>     #affinity_type='hardish3', time_span='3m', update_rule='pairwise+distribute', deterministic=True
>>> )
>>> idxs = self.sample()
>>> idxs = self.sample()
>>> # xdoctest: +REQUIRES(--show)
>>> import kwplot
>>> plt = kwplot.autoplt()
>>> self.show_summary(samples_per_frame=1, fnum=1)
>>> chosen, info = self.show_procedure(fnum=4, idx=10)
>>> plt.subplots_adjust(top=0.9)
>>> kwplot.show_if_requested()

Example

>>> # xdoctest: +REQUIRES(env:SMART_DATA_DVC_DPATH)
>>> from geowatch.tasks.fusion.datamodules.temporal_sampling import *  # NOQA
>>> import geowatch
>>> data_dvc_dpath = geowatch.find_dvc_dpath(tags='phase2_data', hardware='auto')
>>> coco_fpath = data_dvc_dpath / 'Drop6/imganns-KR_R001.kwcoco.zip'
>>> dset = geowatch.coerce_kwcoco(coco_fpath)
>>> vidid = dset.dataset['videos'][0]['id']
>>> self = MultiTimeWindowSampler.from_coco_video(
>>>     dset, vidid,
>>>     time_window=11,
>>>     affinity_type='uniform-soft2-hardish3', update_rule=[''], gamma=2,
>>>     time_span='6m-1y')
>>> self.sample()
>>> # xdoctest: +REQUIRES(--show)
>>> import kwplot
>>> kwplot.autosns()
>>> self.show_summary(3)
>>> kwplot.show_if_requested()

Example

>>> # xdoctest: +SKIP
>>> # TODO: fix the time kernel
>>> # Test under / over sample with time kernels
>>> from geowatch.tasks.fusion.datamodules.temporal_sampling.sampler import *  # NOQA
>>> import geowatch
>>> dset = geowatch.coerce_kwcoco('geowatch-msi', geodata=True, dates=True, num_frames=16, image_size=(8, 8))
>>> vidid = dset.dataset['videos'][0]['id']
>>> self = TimeWindowSampler.from_coco_video(
>>>     dset, vidid,
>>>     time_window=4,
>>>     affinity_type='uniform', update_rule='', time_kernel='-1y,0,1y',
>>> )
>>> self.sample()
class geowatch.tasks.fusion.datamodules.temporal_sampling.sampler.CommonSamplerMixin[source]

Bases: object

classmethod from_coco_video(dset, vidid, gids=None, **kwargs)[source]
classmethod from_datetimes(datetimes, time_span='full', affinity_type='soft2', **kwargs)[source]
class geowatch.tasks.fusion.datamodules.temporal_sampling.sampler.MultiTimeWindowSampler(unixtimes, sensors, time_window=None, affinity_type='hard', update_rule='distribute', deterministic=False, gamma=1, time_span=None, time_kernel=None, name='?', allow_fewer=True)[source]

Bases: CommonSamplerMixin

A wrapper that contains multiple time window samplers with different affinity matrices to increase the diversity of temporal sampling.

Example

>>> from geowatch.tasks.fusion.datamodules.temporal_sampling.sampler import *  # NOQA
>>> import datetime as datetime_mod
>>> from datetime import datetime as datetime_cls
>>> low = datetime_cls.now().timestamp()
>>> high = low + datetime_mod.timedelta(days=365 * 5).total_seconds()
>>> rng = kwarray.ensure_rng(0)
>>> unixtimes = np.array(sorted(rng.randint(low, high, 32)), dtype=float)
>>> sensors = ['a' for _ in range(len(unixtimes))]
>>> time_window = 5
>>> self = MultiTimeWindowSampler(
>>>     unixtimes=unixtimes, sensors=sensors, time_window=time_window, update_rule='pairwise+distribute',
>>>     #time_span=['2y', '1y', '5m'])
>>>     time_span='7d-1m',
>>>     affinity_type='soft2')
>>> self.sample()
>>> # xdoctest: +REQUIRES(--show)
>>> import kwplot
>>> kwplot.autosns()
>>> self.show_summary(10)

Example

>>> from geowatch.tasks.fusion.datamodules.temporal_sampling.sampler import *  # NOQA
>>> import datetime as datetime_mod
>>> from datetime import datetime as datetime_cls
>>> low = datetime_cls.now().timestamp()
>>> high = low + datetime_mod.timedelta(days=365 * 5).total_seconds()
>>> rng = kwarray.ensure_rng(0)
>>> unixtimes = np.array(sorted(rng.randint(low, high, 32)), dtype=float)
>>> sensors = ['a' for _ in range(len(unixtimes))]
>>> time_window = 5
>>> self = MultiTimeWindowSampler(
>>>     unixtimes=unixtimes, sensors=sensors, time_window=time_window, update_rule='distribute',
>>>     time_kernel=['-1y,-3m,0,3m,+1y', '-1m,-1d,0,1d,1m'],
>>>     affinity_type='soft2')
>>> self.sample()
>>> # xdoctest: +REQUIRES(--show)
>>> import kwplot
>>> kwplot.autosns()
>>> self.show_summary(10, show_indexes=1, fnum=1)
>>> list(self.sub_samplers.values())[0].show_summary(10, show_indexes=1, fnum=2)
>>> list(self.sub_samplers.values())[1].show_summary(10, show_indexes=1, fnum=3)

self.subplots_adjust

Parameters:

  • time_span (List[List[str]]) – a list of time spans. e.g. [‘2y’, ‘1y’, ‘5m’]

  • time_kernel (List[str]) – a list of time kernels.

sample(main_frame_idx=None, include=None, exclude=None, return_info=False, error_level=0, rng=None)[source]

Chooses a sub-sampler and samples from it.

Parameters:

  • main_frame_idx (int) – “main” sample index.

  • include (List[int]) – other indexes forced to be included

  • exclude (List[int]) – other indexes forced to be excluded

  • return_info (bool) – for debugging / introspection

  • error_level (int) – See affinity_sample().

Returns:

ndarray | Tuple[ndarray, Dict]

property affinity

Approximate combined affinity, for this multi-sampler

show_summary(samples_per_frame=1, show_indexes=0, fnum=1)[source]

Similar to TimeWindowSampler.show_summary()

class geowatch.tasks.fusion.datamodules.temporal_sampling.sampler.TimeWindowSampler(unixtimes, sensors, time_window=None, affinity_type='hard', update_rule='distribute', deterministic=False, gamma=1, time_span=None, time_kernel=None, affkw=None, name='?', allow_fewer=True)[source]

Bases: CommonSamplerMixin

Object oriented API to produce random temporal samples given a set of keyframes with metadata.

This works by computing a pairwise “affinity” NxN matrix for each of the N keyframes. The details of the affinity matrix depend on parameters passed to this object. Intuitively, the value at Affinity[i, j] represents how much frame-i “wants” to be in the same sample as frame-j.

Parameters:

  • unixtimes (List[int]) – list of unix timestamps for each frame

  • sensors (List[str]) – list of attributes for each frame

  • time_window (int) – number of frames to sample

  • affinity_type (str) – Method for computing the affinity matrix for the underlying sampling algorithm. Can be:

    “soft” - The old generalized random affinity matrix. “soft2” - The new generalized random affinity matrix. “soft3” - The newer generalized random affinity matrix. “hard” - A simplified affinity algorithm. “hardish” - Like hard, but with a blur. “contiguous” - Neighboring frames get high affinity.

  • update_rule (str) – “+” separated string that can contain {“distribute”, “pairwise”}. See affinity_sample() for details.

  • gamma (float) – Modulates sampling probability. Higher values See affinity_sample() for details.

  • time_span (Coercible[datetime.timedelta]) – The ideal distince in time that frames should be separated in. This is typically a string code. E.g. “1y” is one year.

  • name (str) – A name for this object. For developer convinience, has no influence on the algorithm.

  • deterministic (bool) – if True, on each step we choose the next timestamp with maximum probability. Otherwise, we randomly choose a timestep, but with probability according to the current distribution. This is an attribute, which can be modified to change behavior (not thread safe).

Variables:

main_indexes

Example

>>> # xdoctest: +REQUIRES(env:SMART_DATA_DVC_DPATH)
>>> import os
>>> from geowatch.tasks.fusion.datamodules.temporal_sampling.sampler import *  # NOQA
>>> import kwcoco
>>> import geowatch
>>> data_dvc_dpath = geowatch.find_dvc_dpath(tags='phase2_data', hardware='auto')
>>> coco_fpath = data_dvc_dpath / 'Drop6/data_vali_split1.kwcoco.zip'
>>> dset = kwcoco.CocoDataset(coco_fpath)
>>> vidid = dset.dataset['videos'][0]['id']
>>> self = TimeWindowSampler.from_coco_video(
>>>     dset, vidid,
>>>     time_window=5,
>>>     affinity_type='hardish3', time_span='1y',
>>>     update_rule='distribute')
>>> self.deterministic = False
>>> self.show_summary(samples_per_frame=1, fnum=1)
>>> self.deterministic = True
>>> self.show_summary(samples_per_frame=3, fnum=2)
update_affinity(affinity_type=None, update_rule=None)[source]

Construct the affinity matrix given the current affinity_type.

Example

>>> # xdoctest: +REQUIRES(env:SMART_DATA_DVC_DPATH)
>>> import os
>>> from geowatch.tasks.fusion.datamodules.temporal_sampling.sampler import *  # NOQA
>>> import geowatch
>>> data_dvc_dpath = geowatch.find_dvc_dpath(tags='phase2_data', hardware='auto')
>>> coco_fpath = dvc_dpath / 'Drop6/data_vali_split1.kwcoco.zip'
>>> dset = kwcoco.CocoDataset(coco_fpath)
>>> vidid = dset.dataset['videos'][0]['id']
>>> self = TimeWindowSampler.from_coco_video(
>>>     dset, vidid,
>>>     time_window=5,
>>>     affinity_type='contiguous',
>>>     update_rule='pairwise')
>>> self.deterministic = True
>>> self.show_procedure(fnum=1)
compute_affinity(affinity_type=None, update_rule=None)

Construct the affinity matrix given the current affinity_type.

Example

>>> # xdoctest: +REQUIRES(env:SMART_DATA_DVC_DPATH)
>>> import os
>>> from geowatch.tasks.fusion.datamodules.temporal_sampling.sampler import *  # NOQA
>>> import geowatch
>>> data_dvc_dpath = geowatch.find_dvc_dpath(tags='phase2_data', hardware='auto')
>>> coco_fpath = dvc_dpath / 'Drop6/data_vali_split1.kwcoco.zip'
>>> dset = kwcoco.CocoDataset(coco_fpath)
>>> vidid = dset.dataset['videos'][0]['id']
>>> self = TimeWindowSampler.from_coco_video(
>>>     dset, vidid,
>>>     time_window=5,
>>>     affinity_type='contiguous',
>>>     update_rule='pairwise')
>>> self.deterministic = True
>>> self.show_procedure(fnum=1)
property main_indexes
sample(main_frame_idx=None, include=None, exclude=None, return_info=False, error_level=0, rng=None)[source]
Parameters:

  • main_frame_idx (int) – “main” sample index.

  • include (List[int]) – other indexes forced to be included

  • exclude (List[int]) – other indexes forced to be excluded

  • return_info (bool) – for debugging / introspection

  • error_level (int) – See affinity_sample().

Returns:

ndarray | Tuple[ndarray, Dict]

Example

>>> # xdoctest: +REQUIRES(env:SMART_DATA_DVC_DPATH)
>>> import os
>>> import kwcoco
>>> from geowatch.tasks.fusion.datamodules.temporal_sampling.sampler import *  # NOQA
>>> from geowatch.utils.util_data import find_dvc_dpath
>>> dvc_dpath = find_dvc_dpath()
>>> coco_fpath = dvc_dpath / 'Drop2-Aligned-TA1-2022-02-15/data.kwcoco.json'
>>> dset = kwcoco.CocoDataset(coco_fpath)
>>> vidid = dset.dataset['videos'][0]['id']
>>> self = TimeWindowSampler.from_coco_video(
>>>     dset, vidid,
>>>     time_span='1y',
>>>     time_window=3,
>>>     affinity_type='soft2',
>>>     update_rule='distribute+pairwise')
>>> self.deterministic = False
>>> self.show_summary(samples_per_frame=1 if self.deterministic else 10, fnum=1)
>>> self.show_procedure(fnum=2)

Example

>>> import os
>>> import kwcoco
>>> from geowatch.tasks.fusion.datamodules.temporal_sampling.sampler import *  # NOQA
>>> import geowatch
>>> dset = geowatch.coerce_kwcoco('geowatch-msi', geodata=True, dates=True, num_frames=32, image_size=(32, 32), num_videos=1)
>>> vidid = dset.dataset['videos'][0]['id']
>>> self = TimeWindowSampler.from_coco_video(
>>>     dset, vidid,
>>>     time_span='1y',
>>>     time_window=3,
>>>     affinity_type='soft2',
>>>     update_rule='distribute+pairwise')
>>> self.deterministic = True
>>> # xdoctest: +REQUIRES(--show)
>>> self.show_summary(samples_per_frame=1 if self.deterministic else 10, fnum=1)
>>> self.show_procedure(fnum=2)
show_summary(samples_per_frame=1, fnum=1, show_indexes=False, with_temporal=True, compare_determ=True, title_suffix='')[source]

Visualize the affinity matrix and two views of a selected sample.

Plots a figure with three subfigures.

  1. The affinity matrix.

(2) A visualization of a random sampled over “index-space”. A matrix M, where each row is a sample index, each column is a timestep, M[i,j] = 1 (the cell is colored white) to indicate that a sample-i includes timestep-j.

(3) A visualization of the same random sample over “time-space”. A plot where x is the time-axis is drawn, and vertical lines indicate the selectable time indexes. For each sample, a horizontal line indicates the timespan of the sample and an “x” denotes exactly which timesteps are included in that sample.

Example

>>> # xdoctest: +REQUIRES(env:SMART_DATA_DVC_DPATH)
>>> import os
>>> from geowatch.tasks.fusion.datamodules.temporal_sampling import *  # NOQA
>>> from geowatch.utils.util_data import find_dvc_dpath
>>> import kwcoco
>>> # xdoctest: +REQUIRES(--show)
>>> dvc_dpath = find_dvc_dpath()
>>> coco_fpath = dvc_dpath / 'Drop6/data_vali_split1.kwcoco.zip'
>>> dset = kwcoco.CocoDataset(coco_fpath)
>>> video_ids = list(ub.sorted_vals(dset.index.vidid_to_gids, key=len).keys())
>>> vidid = video_ids[2]
>>> # Demo behavior over a grid of parameters
>>> grid = list(ub.named_product({
>>>     'affinity_type': ['hard', 'soft2', 'hardish3', 'hardish2'],
>>>     'update_rule': ['distribute', 'pairwise+distribute'][0:1],
>>>     #'deterministic': [False, True],
>>>     'deterministic': [False],
>>>     'time_window': [5],
>>> }))
>>> import kwplot
>>> kwplot.autompl()
>>> for idx, kwargs in enumerate(grid):
>>>     print('kwargs = {!r}'.format(kwargs))
>>>     self = TimeWindowSampler.from_coco_video(dset, vidid, **kwargs)
>>>     self.show_summary(samples_per_frame=30, fnum=idx, show_indexes=False, deterministic=True)
show_affinity(fnum=3)[source]

Simple drawing of the affinity matrix.

show_procedure(idx=None, exclude=None, fnum=2, rng=None)[source]

Draw a figure that shows the process of performing on call to TimeWindowSampler.sample(). Each row illustrates an iteration of the algorithm. The left column draws the current indicies included in the sample and the right column draws how that sample (corresponding to the current row) influences the probability distribution for the next row.

Example

>>> # xdoctest: +REQUIRES(env:SMART_DATA_DVC_DPATH)
>>> import os
>>> from geowatch.tasks.fusion.datamodules.temporal_sampling import *  # NOQA
>>> from geowatch.utils.util_data import find_dvc_dpath
>>> dvc_dpath = find_dvc_dpath()
>>> coco_fpath = dvc_dpath / 'Drop1-Aligned-L1-2022-01/data.kwcoco.json'
>>> dset = kwcoco.CocoDataset(coco_fpath)
>>> vidid = dset.dataset['videos'][0]['id']
>>> self = TimeWindowSampler.from_coco_video(
>>>     dset, vidid,
>>>     time_window=5,
>>>     affinity_type='soft2',
>>>     update_rule='distribute+pairwise')
>>> self.deterministic = False
>>> self.show_procedure(idx=0, fnum=10)
>>> self.show_affinity(fnum=100)
for idx in xdev.InteractiveIter(list(range(self.num_frames))):

self.show_procedure(idx=idx, fnum=1) xdev.InteractiveIter.draw()

self = TimeWindowSampler.from_coco_video(dset, vidid, time_window=5, affinity_type=’soft2’, update_rule=’distribute+pairwise’) self.deterministic = True self.show_summary(samples_per_frame=20, fnum=1) self.deterministic = False self.show_summary(samples_per_frame=20, fnum=2)

self = TimeWindowSampler.from_coco_video(dset, vidid, time_window=5, affinity_type=’hard’, update_rule=’distribute’) self.deterministic = True self.show_summary(samples_per_frame=20, fnum=3) self.deterministic = False self.show_summary(samples_per_frame=20, fnum=4)

self = TimeWindowSampler.from_coco_video(dset, vidid, time_window=5, affinity_type=’hardish’, update_rule=’distribute’) self.deterministic = True self.show_summary(samples_per_frame=20, fnum=5) self.deterministic = False self.show_summary(samples_per_frame=20, fnum=6)

>>> self.show_procedure(fnum=1)
>>> self.deterministic = True
>>> self.show_procedure(fnum=2)
>>> self.show_procedure(fnum=3)
>>> self.show_procedure(fnum=4)
>>> self.deterministic = False
>>> self.show_summary(samples_per_frame=3, fnum=10)