geowatch.mlops.smart_pipeline module¶

Define the individual nodes that can be composed in a SMART pipeline.

The topology of the pipeline will define the resulting filesystem structure used to store results.

CommandLine

xdoctest -m geowatch.mlops.smart_pipeline __doc__:0
GEOWATCH_DEVCHECK=1 xdoctest -m geowatch.mlops.smart_pipeline __doc__:1

Example

>>> # xdoctest: +SKIP
>>> from geowatch.mlops.smart_pipeline import *  # NOQA
>>> from cmd_queue.util import util_networkx
>>> #
>>> config = {
>>>     'bas_pxl.package_fpath': '/global/models/bas_model2.pt',
>>>     'bas_pxl.num_workers': 3,
>>>     'bas_pxl.tta_time': 1,
>>>     'bas_pxl.test_dataset': '/global/datasets/foobar.kwcoco.zip',
>>> #
>>>     'bas_poly.thresh': 0.1,
>>>     'bas_poly.moving_window_size': 0.1,
>>> #
>>>     'sc_pxl.package_fpath': '/global/models/sc_model2.pt',
>>>     'sc_pxl.tta_fliprot': 8,
>>>     'sc_poly.use_viterbi': 0,
>>> }
>>> #
>>> dag = make_smart_pipeline('joint_bas_sc')
>>> # dag = make_smart_pipeline('bas')
>>> # dag = make_smart_pipeline('sc')
>>> dag.configure(config, root_dpath='/dag-root/dag-id')
>>> #
>>> for node in dag.nodes.values():
>>>     print('---')
>>>     print(f'node={type(node)}')
>>>     print(f'{node.name=}')
>>>     print(f'{node.config=}')
>>>     print(f'{node.in_paths=}')
>>>     print(f'{node.out_paths=}')
>>>     #print(f'{node.resources=}')
>>>     print(f'{node.algo_params=}')
>>>     print('node.depends = {}'.format(ub.urepr(node.depends, nl=1, sort=0)))
>>>     final = node._finalize_templates()
>>>     print('final = {}'.format(ub.urepr(final, nl=2)))
>>>     print('---')
>>> dag.print_graphs()
>>> print('dag.config = {}'.format(ub.urepr(dag.config, nl=1)))
>>> dag_templates = {}
>>> dag_paths = {}
>>> for node in dag.nodes.values():
>>>     dag_templates[node.name] = node._build_templates()['node_dpath']
>>>     dag_paths[node.name] = node._finalize_templates()['node_dpath']
>>>     print(node.command())
>>> import rich
>>> rich.print('dag_templates = {}'.format(
>>>     ub.urepr(dag_templates, nl=1, sv=1, align=':', sort=0)))
>>> rich.print('dag_paths = {}'.format(
>>>     ub.urepr(dag_paths, nl=1, sv=1, align=':', sort=0)))

Example

>>> # xdoctest: +REQUIRES(env:GEOWATCH_DEVCHECK)
>>> from geowatch.mlops.smart_pipeline import *  # NOQA
>>> import geowatch
>>> expt_dvc_dpath = geowatch.find_dvc_dpath(tags='phase2_expt', hardware='auto')
>>> data_dvc_dpath = geowatch.find_dvc_dpath(tags='phase2_data', hardware='auto')
>>> #
>>> config = {}
>>> config['bas_pxl.test_dataset'] = data_dvc_dpath / 'Drop4-BAS/KR_R001.kwcoco.zip'
>>> #config['bas_pxl.test_dataset'] = data_dvc_dpath / 'Drop4-BAS/KR_R002.kwcoco.zip'
>>> #config['bas_pxl.test_dataset'] = data_dvc_dpath / 'Drop4-BAS/BR_R001.kwcoco.zip'
>>> config['bas_pxl.package_fpath'] = expt_dvc_dpath / 'models/fusion/Drop4-BAS/packages/Drop4_TuneV323_BAS_30GSD_BGRNSH_V2/package_epoch0_step41.pt.pt'
>>> config['bas_pxl.num_workers'] = 6
>>> #config['bas_pxl.chip_dims'] = "512,512"
>>> #config['bas_pxl.time_span'] = "1m"
>>> #config['bas_pxl.time_sampling'] = "hardish2"
>>> #config['bas_pxl.use_cloudmask'] = 0
>>> #config['bas_pxl.set_cover_algo'] = 'approx'
>>> #config['bas_pxl.resample_invalid_frames'] = 0
>>> config['bas_poly.thresh'] = 0.1
>>> #config['sc_pxl.chip_dims'] = "256,256"
>>> #config['sc_pxl.use_cloudmask'] = 0
>>> #config['sc_pxl.set_cover_algo'] = 'approx'
>>> #config['sc_pxl.resample_invalid_frames'] = 0
>>> config['sc_pxl.num_workers'] = 6
>>> config['sc_pxl.package_fpath'] = expt_dvc_dpath / 'models/fusion/Drop4-SC/packages/Drop4_tune_V30_8GSD_V3/Drop4_tune_V30_8GSD_V3_epoch=2-step=17334.pt.pt'
>>> #
>>> root_dpath = data_dvc_dpath / '_testdag'
>>> #
>>> nodes = make_smart_pipeline_nodes()
>>> #nodes = bas_nodes()
>>> from geowatch.mlops.pipeline_nodes import PipelineDAG
>>> self = dag = PipelineDAG(nodes)
>>> dag.configure(config=config, root_dpath=root_dpath)
>>> dag.print_graphs()
>>> cmd_queue = dag.submit_jobs()
>>> cmd_queue.write_network_text()
>>> cmd_queue.rprint()
>>> #cmd_queue.run()

class geowatch.mlops.smart_pipeline.FeatureComputation(*, name=None, executable=None, algo_params=None, perf_params=None, in_paths=None, out_paths=None, group_dname=None, root_dpath=None, config=None, node_dpath=None, group_dpath=None, primary_out_key=None, _overwrite_node_dpath=None, _overwrite_group_dpath=None, _no_outarg=False, _no_inarg=False, **aliases)[source]¶

Bases: ProcessNode

executable: str | None = 'python -m geowatch.cli.run_metrics_framework'¶

group_dname: str | None = 'pred'¶

in_paths: Collection = {'src'}¶

command()[source]¶

property condensed¶

class geowatch.mlops.smart_pipeline.FeatureUnion(*, name=None, executable=None, algo_params=None, perf_params=None, in_paths=None, out_paths=None, group_dname=None, root_dpath=None, config=None, node_dpath=None, group_dpath=None, primary_out_key=None, _overwrite_node_dpath=None, _overwrite_group_dpath=None, _no_outarg=False, _no_inarg=False, **aliases)[source]¶

Bases: ProcessNode

name: str | None = 'featunion'¶

executable: str | None = 'geowatch feature_union'¶

group_dname: str | None = 'pred'¶

in_paths: Collection = {'src'}¶

out_paths: Collection = {'dst': 'combo_{featunion_id}.kwcoco.zip'}¶

command()[source]¶

class geowatch.mlops.smart_pipeline.HeatmapPrediction(*, name=None, executable=None, algo_params=None, perf_params=None, in_paths=None, out_paths=None, group_dname=None, root_dpath=None, config=None, node_dpath=None, group_dpath=None, primary_out_key=None, _overwrite_node_dpath=None, _overwrite_group_dpath=None, _no_outarg=False, _no_inarg=False, **aliases)[source]¶

Bases: ProcessNode

executable: str | None = 'python -m geowatch.tasks.fusion.predict'¶

group_dname: str | None = 'pred'¶

perf_params: Collection = {'batch_size': 1, 'devices': '0,', 'num_workers': 2}¶

in_paths: Collection = {'package_fpath', 'test_dataset'}¶

algo_params: Collection = {'drop_unused_frames': True, 'with_change': 'auto', 'with_class': 'auto', 'with_saliency': 'auto'}¶

out_paths: Collection = {'pred_pxl_fpath': 'pred.kwcoco.zip'}¶

command()[source]¶

class geowatch.mlops.smart_pipeline.PolygonPrediction(*, name=None, executable=None, algo_params=None, perf_params=None, in_paths=None, out_paths=None, group_dname=None, root_dpath=None, config=None, node_dpath=None, group_dpath=None, primary_out_key=None, _overwrite_node_dpath=None, _overwrite_group_dpath=None, _no_outarg=False, _no_inarg=False, **aliases)[source]¶

Bases: ProcessNode

executable: str | None = 'python -m geowatch.cli.run_tracker'¶

group_dname: str | None = 'pred'¶

default_track_fn = NotImplemented¶

in_paths: Collection = {'boundary_region', 'pred_pxl_fpath', 'site_summary'}¶

out_paths: Collection = {'poly_kwcoco_fpath': 'poly.kwcoco.zip', 'site_summaries_dpath': 'site_summaries', 'site_summaries_fpath': 'site_summaries_manifest.json', 'sites_dpath': 'sites', 'sites_fpath': 'sites_manifest.json'}¶

command()[source]¶

class geowatch.mlops.smart_pipeline.PolygonEvaluation(*, name=None, executable=None, algo_params=None, perf_params=None, in_paths=None, out_paths=None, group_dname=None, root_dpath=None, config=None, node_dpath=None, group_dpath=None, primary_out_key=None, _overwrite_node_dpath=None, _overwrite_group_dpath=None, _no_outarg=False, _no_inarg=False, **aliases)[source]¶

Bases: ProcessNode

executable: str | None = 'python -m geowatch.cli.run_metrics_framework'¶

group_dname: str | None = 'eval'¶

in_paths: Collection = {'sites_fpath', 'true_region_dpath', 'true_site_dpath'}¶

out_paths: Collection = {'eval_dpath': '.', 'eval_fpath': 'poly_eval.json'}¶

perf_params: Collection = {'enable_viz': False}¶

command()[source]¶

class geowatch.mlops.smart_pipeline.HeatmapEvaluation(*, name=None, executable=None, algo_params=None, perf_params=None, in_paths=None, out_paths=None, group_dname=None, root_dpath=None, config=None, node_dpath=None, group_dpath=None, primary_out_key=None, _overwrite_node_dpath=None, _overwrite_group_dpath=None, _no_outarg=False, _no_inarg=False, **aliases)[source]¶

Bases: ProcessNode

executable: str | None = 'python -m geowatch.tasks.fusion.evaluate'¶

group_dname: str | None = 'eval'¶

in_paths: Collection = {'pred_pxl_fpath', 'true_dataset'}¶

out_paths: Collection = {'eval_pxl_dpath': '.', 'eval_pxl_fpath': 'pxl_eval.json'}¶

command()[source]¶

class geowatch.mlops.smart_pipeline.KWCocoVisualization(*, name=None, executable=None, algo_params=None, perf_params=None, in_paths=None, out_paths=None, group_dname=None, root_dpath=None, config=None, node_dpath=None, group_dpath=None, primary_out_key=None, _overwrite_node_dpath=None, _overwrite_group_dpath=None, _no_outarg=False, _no_inarg=False, **aliases)[source]¶

Bases: ProcessNode

executable: str | None = 'python -m geowatch.cli.coco_visualize_videos'¶

group_dname: str | None = 'pred'¶

in_paths: Collection = {'poly_kwcoco_fpath'}¶

out_paths: Collection = {'viz_dpath': '.', 'viz_stamp_fpath': '_viz.stamp'}¶

command()[source]¶

class geowatch.mlops.smart_pipeline.InvariantFeatureComputation(*, name=None, executable=None, algo_params=None, perf_params=None, in_paths=None, out_paths=None, group_dname=None, root_dpath=None, config=None, node_dpath=None, group_dpath=None, primary_out_key=None, _overwrite_node_dpath=None, _overwrite_group_dpath=None, _no_outarg=False, _no_inarg=False, **aliases)[source]¶

Bases: FeatureComputation

name: str | None = 'invar_feat'¶

out_paths: Collection = {'dst': 'feat_I_{invar_feat_id}.kwcoco.zip'}¶

class geowatch.mlops.smart_pipeline.MaterialFeatureComputation(*, name=None, executable=None, algo_params=None, perf_params=None, in_paths=None, out_paths=None, group_dname=None, root_dpath=None, config=None, node_dpath=None, group_dpath=None, primary_out_key=None, _overwrite_node_dpath=None, _overwrite_group_dpath=None, _no_outarg=False, _no_inarg=False, **aliases)[source]¶

Bases: FeatureComputation

name: str | None = 'mat_feat'¶

out_paths: Collection = {'dst': 'feat_M_{mat_feat_id}.kwcoco.zip'}¶

class geowatch.mlops.smart_pipeline.LandcoverFeatureComputation(*, name=None, executable=None, algo_params=None, perf_params=None, in_paths=None, out_paths=None, group_dname=None, root_dpath=None, config=None, node_dpath=None, group_dpath=None, primary_out_key=None, _overwrite_node_dpath=None, _overwrite_group_dpath=None, _no_outarg=False, _no_inarg=False, **aliases)[source]¶

Bases: FeatureComputation

name: str | None = 'land_feat'¶

out_paths: Collection = {'dst': 'feat_L_{land_feat_id}.kwcoco.zip'}¶

class geowatch.mlops.smart_pipeline.BAS_HeatmapPrediction(*, name=None, executable=None, algo_params=None, perf_params=None, in_paths=None, out_paths=None, group_dname=None, root_dpath=None, config=None, node_dpath=None, group_dpath=None, primary_out_key=None, _overwrite_node_dpath=None, _overwrite_group_dpath=None, _no_outarg=False, _no_inarg=False, **aliases)[source]¶

Bases: HeatmapPrediction

CommandLine

xdoctest -m geowatch.mlops.smart_pipeline BAS_HeatmapPrediction

Example

>>> from geowatch.mlops.smart_pipeline import *  # NOQA
>>> node = BAS_HeatmapPrediction()
>>> node.configure({
>>>     'tta_time': 2,
>>>     'package_fpath': 'foo.pt',
>>>     'test_dataset': 'bar.json',
>>> })
>>> command = node.command()
>>> assert 'tta_time=2' in command
>>> print(command)

name: str | None = 'bas_pxl'¶

algo_params: Collection = {'drop_unused_frames': True, 'with_change': False, 'with_class': False, 'with_saliency': True}¶

property condensed¶

class geowatch.mlops.smart_pipeline.SC_HeatmapPrediction(*, name=None, executable=None, algo_params=None, perf_params=None, in_paths=None, out_paths=None, group_dname=None, root_dpath=None, config=None, node_dpath=None, group_dpath=None, primary_out_key=None, _overwrite_node_dpath=None, _overwrite_group_dpath=None, _no_outarg=False, _no_inarg=False, **aliases)[source]¶

Bases: HeatmapPrediction

name: str | None = 'sc_pxl'¶

algo_params: Collection = {'drop_unused_frames': True, 'saliency_chan_code': 'ac_salient', 'with_change': False, 'with_class': True, 'with_saliency': True}¶

property condensed¶

class geowatch.mlops.smart_pipeline.BAS_PolygonPrediction(*, name=None, executable=None, algo_params=None, perf_params=None, in_paths=None, out_paths=None, group_dname=None, root_dpath=None, config=None, node_dpath=None, group_dpath=None, primary_out_key=None, _overwrite_node_dpath=None, _overwrite_group_dpath=None, _no_outarg=False, _no_inarg=False, **aliases)[source]¶

Bases: PolygonPrediction

name: str | None = 'bas_poly'¶

default_track_fn = 'saliency_heatmaps'¶

property final_algo_config¶

class geowatch.mlops.smart_pipeline.SC_PolygonPrediction(*, name=None, executable=None, algo_params=None, perf_params=None, in_paths=None, out_paths=None, group_dname=None, root_dpath=None, config=None, node_dpath=None, group_dpath=None, primary_out_key=None, _overwrite_node_dpath=None, _overwrite_group_dpath=None, _no_outarg=False, _no_inarg=False, **aliases)[source]¶

Bases: PolygonPrediction

name: str | None = 'sc_poly'¶

default_track_fn = 'class_heatmaps'¶

property final_algo_config¶

class geowatch.mlops.smart_pipeline.BAS_HeatmapEvaluation(*, name=None, executable=None, algo_params=None, perf_params=None, in_paths=None, out_paths=None, group_dname=None, root_dpath=None, config=None, node_dpath=None, group_dpath=None, primary_out_key=None, _overwrite_node_dpath=None, _overwrite_group_dpath=None, _no_outarg=False, _no_inarg=False, **aliases)[source]¶

Bases: HeatmapEvaluation

name: str | None = 'bas_pxl_eval'¶

class geowatch.mlops.smart_pipeline.SC_HeatmapEvaluation(*, name=None, executable=None, algo_params=None, perf_params=None, in_paths=None, out_paths=None, group_dname=None, root_dpath=None, config=None, node_dpath=None, group_dpath=None, primary_out_key=None, _overwrite_node_dpath=None, _overwrite_group_dpath=None, _no_outarg=False, _no_inarg=False, **aliases)[source]¶

Bases: HeatmapEvaluation

name: str | None = 'sc_pxl_eval'¶

class geowatch.mlops.smart_pipeline.BAS_PolygonEvaluation(*, name=None, executable=None, algo_params=None, perf_params=None, in_paths=None, out_paths=None, group_dname=None, root_dpath=None, config=None, node_dpath=None, group_dpath=None, primary_out_key=None, _overwrite_node_dpath=None, _overwrite_group_dpath=None, _no_outarg=False, _no_inarg=False, **aliases)[source]¶

Bases: PolygonEvaluation

name: str | None = 'bas_poly_eval'¶

class geowatch.mlops.smart_pipeline.SC_PolygonEvaluation(*, name=None, executable=None, algo_params=None, perf_params=None, in_paths=None, out_paths=None, group_dname=None, root_dpath=None, config=None, node_dpath=None, group_dpath=None, primary_out_key=None, _overwrite_node_dpath=None, _overwrite_group_dpath=None, _no_outarg=False, _no_inarg=False, **aliases)[source]¶

Bases: PolygonEvaluation

name: str | None = 'sc_poly_eval'¶

class geowatch.mlops.smart_pipeline.BAS_Visualization(*, name=None, executable=None, algo_params=None, perf_params=None, in_paths=None, out_paths=None, group_dname=None, root_dpath=None, config=None, node_dpath=None, group_dpath=None, primary_out_key=None, _overwrite_node_dpath=None, _overwrite_group_dpath=None, _no_outarg=False, _no_inarg=False, **aliases)[source]¶

Bases: KWCocoVisualization

name: str | None = 'bas_poly_viz'¶

class geowatch.mlops.smart_pipeline.SC_Visualization(*, name=None, executable=None, algo_params=None, perf_params=None, in_paths=None, out_paths=None, group_dname=None, root_dpath=None, config=None, node_dpath=None, group_dpath=None, primary_out_key=None, _overwrite_node_dpath=None, _overwrite_group_dpath=None, _no_outarg=False, _no_inarg=False, **aliases)[source]¶

Bases: KWCocoVisualization

name: str | None = 'sc_poly_viz'¶

class geowatch.mlops.smart_pipeline.Cropping(*, name=None, executable=None, algo_params=None, perf_params=None, in_paths=None, out_paths=None, group_dname=None, root_dpath=None, config=None, node_dpath=None, group_dpath=None, primary_out_key=None, _overwrite_node_dpath=None, _overwrite_group_dpath=None, _no_outarg=False, _no_inarg=False, **aliases)[source]¶

Bases: ProcessNode

Used for both site cropping and validation-cropping

executable: str | None = 'python -m geowatch.cli.align'¶

group_dname: str | None = 'pred'¶

algo_params: Collection = {'context_factor': 1.5, 'convexify_regions': True, 'exclude_sensors': 'L8', 'force_min_gsd': None, 'force_nodata': -9999, 'include_channels': None, 'minimum_size': None, 'rpc_align_method': 'orthorectify', 'target_gsd': 4}¶

perf_params: Collection = {'aux_workers': 4, 'geo_preprop': 'auto', 'img_workers': 32, 'keep': 'img', 'verbose': 1}¶

in_paths: Collection = {'crop_src_fpath', 'regions'}¶

property condensed¶

command()[source]¶

class geowatch.mlops.smart_pipeline.SiteClustering(*, name=None, executable=None, algo_params=None, perf_params=None, in_paths=None, out_paths=None, group_dname=None, root_dpath=None, config=None, node_dpath=None, group_dpath=None, primary_out_key=None, _overwrite_node_dpath=None, _overwrite_group_dpath=None, _no_outarg=False, _no_inarg=False, **aliases)[source]¶

Bases: ProcessNode

Crop to each image of every site.

CommandLine

xdoctest -m geowatch.mlops.smart_pipeline SiteClustering

Example

>>> from geowatch.mlops.smart_pipeline import *  # NOQA
>>> node = SiteClustering()
>>> command = node.final_command()
>>> print(command)

executable: str | None = 'python -m geowatch.cli.cluster_sites'¶

name: str | None = 'cluster_sites'¶

group_dname: str | None = 'crops'¶

algo_params: Collection = {'context_factor': 1.5, 'crop_time': True, 'maximum_size': '1024x1024@2GSD', 'minimum_size': '128x128@2GSD'}¶

perf_params: Collection = {'draw_clusters': False, 'io_workers': 4}¶

in_paths: Collection = {'src'}¶

out_paths: Collection = {'dst_dpath': 'clustered', 'dst_region_fpath': 'clustered.geojson'}¶

class geowatch.mlops.smart_pipeline.SC_Cropping(*, name=None, executable=None, algo_params=None, perf_params=None, in_paths=None, out_paths=None, group_dname=None, root_dpath=None, config=None, node_dpath=None, group_dpath=None, primary_out_key=None, _overwrite_node_dpath=None, _overwrite_group_dpath=None, _no_outarg=False, _no_inarg=False, **aliases)[source]¶

Bases: Cropping

Crop to each image of every site.

CommandLine

xdoctest -m geowatch.mlops.smart_pipeline SC_Cropping

Example

>>> from geowatch.mlops.smart_pipeline import *  # NOQA
>>> node = SC_Cropping()
>>> command = node.command()
>>> print(command)
>>> assert '--regions' in command

name: str | None = 'sc_crop'¶

group_dname: str | None = 'crops'¶

algo_params: Collection = {'context_factor': 1.0, 'convexify_regions': True, 'exclude_sensors': 'L8', 'force_min_gsd': 2, 'force_nodata': -9999, 'include_channels': None, 'minimum_size': '128x128@10GSD', 'rpc_align_method': 'orthorectify', 'target_gsd': 4}¶

out_paths: Collection = {'crop_dst_fpath': 'sitecrop.kwcoco.zip'}¶

class geowatch.mlops.smart_pipeline.SV_Cropping(*, name=None, executable=None, algo_params=None, perf_params=None, in_paths=None, out_paths=None, group_dname=None, root_dpath=None, config=None, node_dpath=None, group_dpath=None, primary_out_key=None, _overwrite_node_dpath=None, _overwrite_group_dpath=None, _no_outarg=False, _no_inarg=False, **aliases)[source]¶

Bases: Cropping

Crop to high res images as the start / end of a sequence

CommandLine

xdoctest -m geowatch.mlops.smart_pipeline SV_Cropping

Example

>>> from geowatch.mlops.smart_pipeline import *  # NOQA
>>> node = SV_Cropping()
>>> command = node.command()
>>> print(command)
>>> assert '--regions' in command

name: str | None = 'sv_crop'¶

algo_params: Collection = {'context_factor': 1.3, 'convexify_regions': True, 'force_min_gsd': 2, 'force_nodata': -9999, 'include_sensors': 'WV', 'minimum_size': '128x128@2GSD', 'num_end_frames': 3, 'num_start_frames': 3, 'rpc_align_method': 'orthorectify', 'target_gsd': 2}¶

out_paths: Collection = {'crop_dst_fpath': 'sv_crop.kwcoco.zip'}¶

class geowatch.mlops.smart_pipeline.SV_DepthPredict(*, name=None, executable=None, algo_params=None, perf_params=None, in_paths=None, out_paths=None, group_dname=None, root_dpath=None, config=None, node_dpath=None, group_dpath=None, primary_out_key=None, _overwrite_node_dpath=None, _overwrite_group_dpath=None, _no_outarg=False, _no_inarg=False, **aliases)[source]¶

Bases: ProcessNode

Node for DZYNEs high res depth-based parallel change detector.

This takes in a kwcoco file with images, and geojson annotations, projects those annotations onto the videos, scores each video / track, and then writes a modified kwcoco file.

Example

>>> from geowatch.mlops import smart_pipeline
>>> self = node = smart_pipeline.SV_DepthPredict(root_dpath='/ROOT/DPATH/')
>>> node.configure({
>>>     'input_kwcoco': 'my_highres.kwcoco.zip',
>>>     'input_region': 'myregion.geojson',
>>>     'input_sites': 'myinput_sites',
>>>     'model_fpath': 'models/depth_pcd/basicModel2.h5',
>>> })
>>> print(node.command())

name: str | None = 'sv_depth_score'¶

executable: str | None = 'python -m geowatch.tasks.depth_pcd.score_tracks'¶

group_dname: str | None = 'pred'¶

in_paths: Collection = {'input_kwcoco', 'input_region'}¶

out_paths: Collection = {'out_kwcoco': 'pred_depth_scores.kwcoco.zip'}¶

algo_params: Collection = {'model_fpath': None}¶

command()[source]¶

class geowatch.mlops.smart_pipeline.SV_DepthFilter(*, name=None, executable=None, algo_params=None, perf_params=None, in_paths=None, out_paths=None, group_dname=None, root_dpath=None, config=None, node_dpath=None, group_dpath=None, primary_out_key=None, _overwrite_node_dpath=None, _overwrite_group_dpath=None, _no_outarg=False, _no_inarg=False, **aliases)[source]¶

Bases: ProcessNode

Node for DZYNEs high res depth-based parallel change detector.

Takes in a scored kwcoco file from SV_DepthPredict and geojson annotations and then filters the annotations based on the scores in the kwcoco file.

Example

>>> from geowatch.mlops import smart_pipeline
>>> self = node = smart_pipeline.SV_DepthFilter(node_dpath='/MY/OUPUT/DIR/')
>>> node.configure({
>>>     'input_kwcoco': 'myscored.kwcoco.zip',
>>>     'input_region': 'myregion.geojson',
>>>     'input_sites': 'mysites/*.geojson',
>>> })
>>> print(node.command())

Example

>>> from geowatch.mlops import smart_pipeline
>>> self = node = smart_pipeline.SV_DepthFilter(root_dpath='/ROOT/DPATH/')
>>> node.configure({
>>>     'input_kwcoco': 'foo.kwcoco',
>>>     'input_region': 'region.geojson',
>>>     'input_sites': 'input_sites',
>>>     #'output_sites_dpath': 'I_WANT_OUT_SITES_HERE',
>>>     'output_region_fpath': 'I_WANT_OUT_REGIONS_HERE',
>>>     'output_site_manifest_fpath': 'I_WANT_SITE_MANIFESTS_HERE',
>>> })
>>> print('self.template_out_paths = {}'.format(ub.urepr(self.template_out_paths, nl=1)))
>>> print('self.final_out_paths = {}'.format(ub.urepr(self.final_out_paths, nl=1)))
>>> print(node.command())

name: str | None = 'sv_depth_filter'¶

executable: str | None = 'python -m geowatch.tasks.depth_pcd.filter_tracks'¶

group_dname: str | None = 'pred'¶

in_paths: Collection = {'input_kwcoco', 'input_region', 'input_sites'}¶

out_paths: Collection = {'output_region_fpath': 'sv_depth_out_region.geojson', 'output_site_manifest_fpath': 'sv_depth_out_site_manifest.json', 'output_sites_dpath': 'sv_depth_out_sites'}¶

algo_params: Collection = {'threshold': 0.4}¶

command()[source]¶

class geowatch.mlops.smart_pipeline.DinoBoxDetector(*, name=None, executable=None, algo_params=None, perf_params=None, in_paths=None, out_paths=None, group_dname=None, root_dpath=None, config=None, node_dpath=None, group_dpath=None, primary_out_key=None, _overwrite_node_dpath=None, _overwrite_group_dpath=None, _no_outarg=False, _no_inarg=False, **aliases)[source]¶

Bases: ProcessNode

Used for both site cropping and validation-cropping

Example

>>> from geowatch.mlops.smart_pipeline import *  # NOQA
>>> node = DinoBoxDetector(root_dpath='/root/dpath/')
>>> node.configure({
>>>     'coco_fpath': 'foo.kwcoco',
>>>     'package_fpath': 'model.pt',
>>>     'data_workers': 2,
>>> })
>>> print(node.command())
>>> algo_id1 = node.algo_id
>>> print(f'node.algo_id={node.algo_id}')
>>> print(f'node.process_id={node.process_id}')
>>> node.configure({
>>>     'coco_fpath': 'foo.kwcoco',
>>>     'package_fpath': 'model.pt',
>>>     'data_workers': 10,
>>> })
>>> algo_id2 = node.algo_id
>>> node.configure({
>>>     'fixed_resolution': "10GSD",
>>> })
>>> algo_id3 = node.algo_id
>>> assert algo_id1 == algo_id2, 'perf params dont change hash'
>>> assert algo_id1 != algo_id3, 'algo params do change hash'

name: str | None = 'sv_dino_boxes'¶

executable: str | None = 'python -m geowatch.tasks.dino_detector.predict'¶

group_dname: str | None = 'pred'¶

in_paths: Collection = {'coco_fpath'}¶

out_paths: Collection = {'out_coco_fpath': 'pred_boxes.kwcoco.zip'}¶

algo_params: Collection = {'batch_size': 1, 'fixed_resolution': '3GSD', 'package_fpath': None, 'window_dims': 256, 'window_overlap': 0.5}¶

perf_params: Collection = {'data_workers': 2, 'device': 0}¶

command()[source]¶

class geowatch.mlops.smart_pipeline.SV_DinoFilter(*, name=None, executable=None, algo_params=None, perf_params=None, in_paths=None, out_paths=None, group_dname=None, root_dpath=None, config=None, node_dpath=None, group_dpath=None, primary_out_key=None, _overwrite_node_dpath=None, _overwrite_group_dpath=None, _no_outarg=False, _no_inarg=False, **aliases)[source]¶

Bases: ProcessNode

Used for both site cropping and validation-cropping

Example

>>> from geowatch.mlops.smart_pipeline import *  # NOQA
>>> self = node = SV_DinoFilter(root_dpath='/ROOT/DPATH/')
>>> node.configure({
>>>     'input_kwcoco': 'foo.kwcoco',
>>>     'input_region': 'region.geojson',
>>>     'input_sites': 'input_sites',
>>>     #'output_sites_dpath': 'I_WANT_OUT_SITES_HERE',
>>>     'output_region_fpath': 'I_WANT_OUT_REGIONS_HERE',
>>>     'output_site_manifest_fpath': 'I_WANT_SITE_MANIFESTS_HERE',
>>> })
>>> print('self.template_out_paths = {}'.format(ub.urepr(self.template_out_paths, nl=1)))
>>> print('self.final_out_paths = {}'.format(ub.urepr(self.final_out_paths, nl=1)))
>>> print(node.command())

name: str | None = 'sv_dino_filter'¶

executable: str | None = 'python -m geowatch.tasks.dino_detector.building_validator'¶

group_dname: str | None = 'pred'¶

in_paths: Collection = {'input_kwcoco', 'input_region', 'input_sites'}¶

out_paths: Collection = {'output_region_fpath': 'out_region.geojson', 'output_site_manifest_fpath': 'out_site_manifest.json', 'output_sites_dpath': 'out_sites'}¶

algo_params: Collection = {'box_isect_threshold': 0.1, 'box_score_threshold': 0.1, 'end_min_score': 0.1, 'start_max_score': 1.0}¶

command()[source]¶

geowatch.mlops.smart_pipeline.bas_nodes()[source]¶

geowatch.mlops.smart_pipeline.sc_nodes()[source]¶

geowatch.mlops.smart_pipeline.make_smart_pipeline_nodes(with_bas=True, building_validation=False, depth_validation=False, site_crops=True, with_acsc=True, site_cluster=False)[source]¶

geowatch.mlops.smart_pipeline.make_smart_pipeline(name)[source]¶

Get an unconfigured instance of the SMART pipeline

CommandLine

xdoctest -m geowatch.mlops.smart_pipeline make_smart_pipeline

Example

>>> from geowatch.mlops.smart_pipeline import *  # NOQA
>>> dag = make_smart_pipeline('sc')
>>> dag.print_graphs()
>>> dag.inspect_configurables()

>>> from geowatch.mlops.smart_pipeline import *  # NOQA
>>> dag = make_smart_pipeline('joint_bas_sc')
>>> dag.print_graphs()
>>> dag.inspect_configurables()

>>> from geowatch.mlops.smart_pipeline import *  # NOQA
>>> dag = make_smart_pipeline('joint_bas_sv_sc')
>>> dag.print_graphs()
>>> dag.inspect_configurables()

>>> from geowatch.mlops.smart_pipeline import *  # NOQA
>>> dag = make_smart_pipeline('full')
>>> dag.print_graphs()
>>> dag.inspect_configurables()

Example

>>> from geowatch.mlops.smart_pipeline import *  # NOQA
>>> dag = make_smart_pipeline('bas_depth_vali')
>>> dag.print_graphs()
>>> dag.inspect_configurables()

>>> from geowatch.mlops.smart_pipeline import *  # NOQA
>>> dag = make_smart_pipeline('dzyne_sv_only')
>>> dag.print_graphs()
>>> dag.inspect_configurables()

geowatch.mlops.smart_pipeline.dzyne_sv_only_pipeline()[source]¶

Demo Schedule Evaluate Inovcation:

HIRES_DVC_DATA_DPATH=$(geowatch_dvc –tags=’drop7_data’ –hardware=auto) DVC_EXPT_DPATH=$(geowatch_dvc –tags=’phase2_expt’ –hardware=auto)

python -m geowatch.mlops.schedule_evaluation –params=”
pipeline: dzyne_sv_only matrix:

sv_depth_score.input_region:
$HIRES_DVC_DATA_DPATH/Drop7-StaticACTestSet-2GSD/bas_small_output/region_models/KR_R002.geojson # $HIRES_DVC_DATA_DPATH/Drop7-StaticACTestSet-2GSD/bas_small_output/region_models/CN_C500.geojson # $HIRES_DVC_DATA_DPATH/Drop7-StaticACTestSet-2GSD/bas_small_output/region_models/CO_C501.geojson # $HIRES_DVC_DATA_DPATH/Drop7-StaticACTestSet-2GSD/bas_small_output/region_models/KW_C501.geojson

sv_depth_score.model_fpath:

$DVC_EXPT_DPATH/models/depth_pcd/basicModel2.h5

# - $DVC_EXPT_DPATH/models/depth_pcd/model3.h5

sv_depth_filter.threshold:

0.1

# - 0.2

sv_poly_eval.true_region_dpath: $HIRES_DVC_DATA_DPATH/Drop7-StaticACTestSet-2GSD/bas_small_truth/region_models sv_poly_eval.true_site_dpath: $HIRES_DVC_DATA_DPATH/Drop7-StaticACTestSet-2GSD/bas_small_truth/site_models pre_poly_eval.true_region_dpath: $HIRES_DVC_DATA_DPATH/Drop7-StaticACTestSet-2GSD/bas_small_truth/region_models pre_poly_eval.true_site_dpath: $HIRES_DVC_DATA_DPATH/Drop7-StaticACTestSet-2GSD/bas_small_truth/site_models

submatrices:
# For each region, pair it with the appropriate input kwcoco

sv_depth_score.input_region: $HIRES_DVC_DATA_DPATH/Drop7-StaticACTestSet-2GSD/bas_small_output/region_models/KR_R002.geojson sv_depth_filter.input_sites: $HIRES_DVC_DATA_DPATH/Drop7-StaticACTestSet-2GSD/bas_small_output/region_models/KR_R002 pre_poly_eval.sites_fpath: $HIRES_DVC_DATA_DPATH/Drop7-StaticACTestSet-2GSD/bas_small_output/region_models/KR_R002 sv_depth_score.input_kwcoco: $HIRES_DVC_DATA_DPATH/Drop7-StaticACTestSet-2GSD/KR_R002/imgonly-KR_R002-rawbands-small.kwcoco.zip

sv_depth_score.input_region: $HIRES_DVC_DATA_DPATH/Drop7-StaticACTestSet-2GSD/bas_small_output/region_models/CN_C500.geojson sv_depth_filter.input_sites: $HIRES_DVC_DATA_DPATH/Drop7-StaticACTestSet-2GSD/bas_small_output/region_models/CN_C500 pre_poly_eval.sites_fpath: $HIRES_DVC_DATA_DPATH/Drop7-StaticACTestSet-2GSD/bas_small_output/region_models/CN_C500 sv_depth_score.input_kwcoco: $HIRES_DVC_DATA_DPATH/Drop7-StaticACTestSet-2GSD/CN_C000/imgonly-CN_C000-rawbands-small.kwcoco.zip

sv_depth_score.input_region: $HIRES_DVC_DATA_DPATH/Drop7-StaticACTestSet-2GSD/bas_small_output/region_models/CO_C501.geojson sv_depth_filter.input_sites: $HIRES_DVC_DATA_DPATH/Drop7-StaticACTestSet-2GSD/bas_small_output/region_models/CO_C501 pre_poly_eval.sites_fpath: $HIRES_DVC_DATA_DPATH/Drop7-StaticACTestSet-2GSD/bas_small_output/region_models/CO_C501 sv_depth_score.input_kwcoco: $HIRES_DVC_DATA_DPATH/Drop7-StaticACTestSet-2GSD/CO_C001/imgonly-CO_C001-rawbands-small.kwcoco.zip

sv_depth_score.input_region: $HIRES_DVC_DATA_DPATH/Drop7-StaticACTestSet-2GSD/bas_small_output/region_models/KW_C501.geojson sv_depth_filter.input_sites: $HIRES_DVC_DATA_DPATH/Drop7-StaticACTestSet-2GSD/bas_small_output/region_models/KW_C501 pre_poly_eval.sites_fpath: $HIRES_DVC_DATA_DPATH/Drop7-StaticACTestSet-2GSD/bas_small_output/region_models/KW_C501 sv_depth_score.input_kwcoco: $HIRES_DVC_DATA_DPATH/Drop7-StaticACTestSet-2GSD/KW_C001/imgonly-KW_C001-rawbands-small.kwcoco.zip

“ –root_dpath=”$DVC_EXPT_DPATH/_test_dzyne_sv_only” –devices=”0,1” –tmux_workers=8 –backend=tmux –queue_name “_test_dzyne_sv_only” –skip_existing=0 –run=0