geowatch.mlops.smart_global_helper module

Quick and dirty project specific stuff that ideally wont get in the way of general use-cases but should eventually be factored out.

Special heuristics. Used by ./aggregate.py and ./aggregate_plots.py

class geowatch.mlops.smart_global_helper.SmartGlobalHelper

Bases: object

A class for SMART-specific hacks and defaults for mlops

Should be stateless

TARGET_METRICS = {'ac_time_delta': '5 days', 'bas_f1': 0.7, 'bas_fpr': 0.1, 'bas_ppv': 0.85, 'bas_tpr': 0.59, 'cost_dollars_per_km2': 30}

VIZ_BLOCKLIST = {'resolved_params.bas_poly.append_mode', 'resolved_params.bas_poly.clear_annots', 'resolved_params.bas_poly.default_track_fn', 'resolved_params.bas_poly.in_file', 'resolved_params.bas_poly.in_file_gt', 'resolved_params.bas_poly.out_kwcoco', 'resolved_params.bas_poly.out_site_summaries_dir', 'resolved_params.bas_poly.out_site_summaries_fpath', 'resolved_params.bas_poly.out_sites_dir', 'resolved_params.bas_poly.out_sites_fpath', 'resolved_params.bas_poly.region_id', 'resolved_params.bas_poly.site_summary', 'resolved_params.bas_poly_eval.enable_viz', 'resolved_params.bas_poly_eval.gt_dpath', 'resolved_params.bas_poly_eval.load_workers', 'resolved_params.bas_poly_eval.merge', 'resolved_params.bas_poly_eval.merge_fbetas', 'resolved_params.bas_poly_eval.merge_fpath', 'resolved_params.bas_poly_eval.name', 'resolved_params.bas_poly_eval.out_dir', 'resolved_params.bas_poly_eval.pred_sites', 'resolved_params.bas_poly_eval.tmp_dir', 'resolved_params.bas_poly_eval.true_region_dpath', 'resolved_params.bas_poly_eval.true_site_dpath', 'resolved_params.bas_poly_eval.use_cache', 'resolved_params.bas_pxl.balance_areas', 'resolved_params.bas_pxl.batch_size', 'resolved_params.bas_pxl.clear_annots', 'resolved_params.bas_pxl.compress', 'resolved_params.bas_pxl.config_file', 'resolved_params.bas_pxl.datamodule', 'resolved_params.bas_pxl.devices', 'resolved_params.bas_pxl.dist_weights', 'resolved_params.bas_pxl.downweight_nan_regions', 'resolved_params.bas_pxl.ignore_dilate', 'resolved_params.bas_pxl.max_epoch_length', 'resolved_params.bas_pxl.min_spacetime_weight', 'resolved_params.bas_pxl.neg_to_pos_ratio', 'resolved_params.bas_pxl.normalize_inputs', 'resolved_params.bas_pxl.num_workers', 'resolved_params.bas_pxl.pred_dataset', 'resolved_params.bas_pxl.quantize', 'resolved_params.bas_pxl.resample_invalid_frames', 'resolved_params.bas_pxl.sqlview', 'resolved_params.bas_pxl.temporal_dropout', 'resolved_params.bas_pxl.test_dataset', 'resolved_params.bas_pxl.torch_sharing_strategy', 'resolved_params.bas_pxl.torch_start_method', 'resolved_params.bas_pxl.track_emissions', 'resolved_params.bas_pxl.train_dataset', 'resolved_params.bas_pxl.upweight_centers', 'resolved_params.bas_pxl.upweight_time', 'resolved_params.bas_pxl.use_centered_positives', 'resolved_params.bas_pxl.use_grid_cache', 'resolved_params.bas_pxl.use_grid_positives', 'resolved_params.bas_pxl.use_grid_valid_regions', 'resolved_params.bas_pxl.vali_dataset', 'resolved_params.bas_pxl.weight_dilate', 'resolved_params.bas_pxl.with_change', 'resolved_params.bas_pxl.with_class', 'resolved_params.bas_pxl.with_saliency', 'resolved_params.bas_pxl.write_out_config_file_to_this_path', 'resolved_params.bas_pxl.write_preds', 'resolved_params.bas_pxl.write_probs', 'resolved_params.bas_pxl.write_workers', 'resolved_params.bas_pxl_eval.balance_area', 'resolved_params.bas_pxl_eval.draw_curves', 'resolved_params.bas_pxl_eval.draw_heatmaps', 'resolved_params.bas_pxl_eval.draw_workers', 'resolved_params.bas_pxl_eval.eval_dpath', 'resolved_params.bas_pxl_eval.eval_fpath', 'resolved_params.bas_pxl_eval.pred_dataset', 'resolved_params.bas_pxl_eval.resolution', 'resolved_params.bas_pxl_eval.score_space', 'resolved_params.bas_pxl_eval.true_dataset', 'resolved_params.bas_pxl_eval.viz_thresh', 'resolved_params.bas_pxl_eval.workers', 'resolved_params.bas_pxl_fit.accelerator', 'resolved_params.bas_pxl_fit.accumulate_grad_batches', 'resolved_params.bas_pxl_fit.datamodule', 'resolved_params.bas_pxl_fit.devices', 'resolved_params.bas_pxl_fit.gradient_clip_algorithm', 'resolved_params.bas_pxl_fit.gradient_clip_val', 'resolved_params.bas_pxl_fit.inference_mode', 'resolved_params.bas_pxl_fit.max_epochs', 'resolved_params.bas_pxl_fit.max_steps', 'resolved_params.bas_pxl_fit.method', 'resolved_params.bas_pxl_fit.name', 'resolved_params.bas_pxl_fit.patience', 'resolved_params.bas_pxl_fit.precision', 'resolved_params.bas_pxl_fit.sqlview', 'resolved_params.bas_pxl_fit.stochastic_weight_avg', 'resolved_params.bas_pxl_fit.use_grid_cache', 'resolved_params.bas_pxl_fit.use_grid_valid_regions'}

EXTRA_HASHID_IGNORE_COLUMNS = ['params.sc_poly.site_summary', 'params.sc_pxl.num_workers', 'params.bas_pxl.num_workers']

EXTRA_PATH_COLUMNS = ['params.bas_poly_eval.true_site_dpath', 'params.bas_poly_eval.true_region_dpath', 'params.bas_poly.boundary_region']

LABEL_MAPPINGS = {'metrics.sc_poly_eval.bas_f1': 'BAS-F1', 'metrics.sc_poly_eval.bas_faa_f1': 'BAS-FAA-F1', 'metrics.sc_poly_eval.bas_ffpa': 'FFPA', 'metrics.sc_poly_eval.sc_macro_f1': 'AC-F1 (macro)', 'region_id': 'Region', 'sc_poly_eval.bas_f1': 'BAS-F1', 'sc_poly_eval.bas_faa_f1': 'BAS-FAA-F1', 'sc_poly_eval.bas_ffpa': 'FFPA', 'sc_poly_eval.sc_macro_f1': 'AC-F1 (macro)'}

shared_palettes(macro_table)

For each key in a hard code set (relevant to SMART), assign a consistent color to those values so our plots are comparble.

make_param_palette(param_values)

region_palette(rois=None)

label_modifier()

Build the label modifier for the SMART task.

Returns:

util_kwplot.LabelModifier

default_vantage_points(eval_type)

mark_star_models(macro_table)

mark_delivery(table, include=None)

self = SMART_HELPER

populate_test_dataset_bundles(agg)

Attempt to parse out which kwcoco bundle test datasets belonged to

get_delivered_model_params()

custom_channel_relabel(sub_macro_table, channel_key, coarsen=False)

custom_channel_relabel_mapping(unique_channels, coarsen=False)

print_minmax_times(table)

threshold_param_groups(table, param_name, metric_name, metric_threshold)