From c31bf85a230d036c89c137e051560b157f7a479e Mon Sep 17 00:00:00 2001 From: Jeremy Reizenstein Date: Thu, 26 May 2022 05:33:03 -0700 Subject: [PATCH] test runner for experiment.py Summary: Add simple interactive testrunner for experiment.py Reviewed By: shapovalov Differential Revision: D35316221 fbshipit-source-id: d424bcba632eef89eefb56e18e536edb58ec6f85 --- projects/implicitron_trainer/experiment.py | 5 +- .../implicitron_trainer/tests/__init__.py | 5 + .../implicitron_trainer/tests/experiment.yaml | 340 ++++++++++++++++++ .../tests/test_experiment.py | 103 ++++++ tests/common_testing.py | 4 +- 5 files changed, 453 insertions(+), 4 deletions(-) create mode 100644 projects/implicitron_trainer/tests/__init__.py create mode 100644 projects/implicitron_trainer/tests/experiment.yaml create mode 100644 projects/implicitron_trainer/tests/test_experiment.py diff --git a/projects/implicitron_trainer/experiment.py b/projects/implicitron_trainer/experiment.py index a888e225..4bf8d57c 100755 --- a/projects/implicitron_trainer/experiment.py +++ b/projects/implicitron_trainer/experiment.py @@ -705,8 +705,9 @@ class ExperimentConfig: ) -cs = hydra.core.config_store.ConfigStore.instance() -cs.store(name="default_config", node=ExperimentConfig) +if __name__ == "__main__": + cs = hydra.core.config_store.ConfigStore.instance() + cs.store(name="default_config", node=ExperimentConfig) @hydra.main(config_path="./configs/", config_name="default_config") diff --git a/projects/implicitron_trainer/tests/__init__.py b/projects/implicitron_trainer/tests/__init__.py new file mode 100644 index 00000000..2e41cd71 --- /dev/null +++ b/projects/implicitron_trainer/tests/__init__.py @@ -0,0 +1,5 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# All rights reserved. +# +# This source code is licensed under the BSD-style license found in the +# LICENSE file in the root directory of this source tree. diff --git a/projects/implicitron_trainer/tests/experiment.yaml b/projects/implicitron_trainer/tests/experiment.yaml new file mode 100644 index 00000000..640fa31b --- /dev/null +++ b/projects/implicitron_trainer/tests/experiment.yaml @@ -0,0 +1,340 @@ +generic_model_args: + mask_images: true + mask_depths: true + render_image_width: 400 + render_image_height: 400 + mask_threshold: 0.5 + output_rasterized_mc: false + bg_color: + - 0.0 + - 0.0 + - 0.0 + num_passes: 1 + chunk_size_grid: 4096 + render_features_dimensions: 3 + tqdm_trigger_threshold: 16 + n_train_target_views: 1 + sampling_mode_training: mask_sample + sampling_mode_evaluation: full_grid + raysampler_class_type: AdaptiveRaySampler + renderer_class_type: MultiPassEmissionAbsorptionRenderer + image_feature_extractor_class_type: null + view_pooler_enabled: false + implicit_function_class_type: NeuralRadianceFieldImplicitFunction + loss_weights: + loss_rgb_mse: 1.0 + loss_prev_stage_rgb_mse: 1.0 + loss_mask_bce: 0.0 + loss_prev_stage_mask_bce: 0.0 + log_vars: + - loss_rgb_psnr_fg + - loss_rgb_psnr + - loss_rgb_mse + - loss_rgb_huber + - loss_depth_abs + - loss_depth_abs_fg + - loss_mask_neg_iou + - loss_mask_bce + - loss_mask_beta_prior + - loss_eikonal + - loss_density_tv + - loss_depth_neg_penalty + - loss_autodecoder_norm + - loss_prev_stage_rgb_mse + - loss_prev_stage_rgb_psnr_fg + - loss_prev_stage_rgb_psnr + - loss_prev_stage_mask_bce + - objective + - epoch + - sec/it + sequence_autodecoder_args: + encoding_dim: 0 + n_instances: 0 + init_scale: 1.0 + ignore_input: false + raysampler_AdaptiveRaySampler_args: + image_width: 400 + image_height: 400 + sampling_mode_training: mask_sample + sampling_mode_evaluation: full_grid + n_pts_per_ray_training: 64 + n_pts_per_ray_evaluation: 64 + n_rays_per_image_sampled_from_mask: 1024 + stratified_point_sampling_training: true + stratified_point_sampling_evaluation: false + scene_extent: 8.0 + scene_center: + - 0.0 + - 0.0 + - 0.0 + raysampler_NearFarRaySampler_args: + image_width: 400 + image_height: 400 + sampling_mode_training: mask_sample + sampling_mode_evaluation: full_grid + n_pts_per_ray_training: 64 + n_pts_per_ray_evaluation: 64 + n_rays_per_image_sampled_from_mask: 1024 + stratified_point_sampling_training: true + stratified_point_sampling_evaluation: false + min_depth: 0.1 + max_depth: 8.0 + renderer_LSTMRenderer_args: + num_raymarch_steps: 10 + init_depth: 17.0 + init_depth_noise_std: 0.0005 + hidden_size: 16 + n_feature_channels: 256 + verbose: false + renderer_MultiPassEmissionAbsorptionRenderer_args: + raymarcher_class_type: EmissionAbsorptionRaymarcher + n_pts_per_ray_fine_training: 64 + n_pts_per_ray_fine_evaluation: 64 + stratified_sampling_coarse_training: true + stratified_sampling_coarse_evaluation: false + append_coarse_samples_to_fine: true + density_noise_std_train: 0.0 + return_weights: false + raymarcher_EmissionAbsorptionRaymarcher_args: + surface_thickness: 1 + bg_color: + - 0.0 + background_opacity: 10000000000.0 + density_relu: true + blend_output: false + raymarcher_CumsumRaymarcher_args: + surface_thickness: 1 + bg_color: + - 0.0 + background_opacity: 0.0 + density_relu: true + blend_output: false + renderer_SignedDistanceFunctionRenderer_args: + render_features_dimensions: 3 + ray_tracer_args: + object_bounding_sphere: 1.0 + sdf_threshold: 5.0e-05 + line_search_step: 0.5 + line_step_iters: 1 + sphere_tracing_iters: 10 + n_steps: 100 + n_secant_steps: 8 + ray_normal_coloring_network_args: + feature_vector_size: 3 + mode: idr + d_in: 9 + d_out: 3 + dims: + - 512 + - 512 + - 512 + - 512 + weight_norm: true + n_harmonic_functions_dir: 0 + pooled_feature_dim: 0 + bg_color: + - 0.0 + soft_mask_alpha: 50.0 + image_feature_extractor_ResNetFeatureExtractor_args: + name: resnet34 + pretrained: true + stages: + - 1 + - 2 + - 3 + - 4 + normalize_image: true + image_rescale: 0.16 + first_max_pool: true + proj_dim: 32 + l2_norm: true + add_masks: true + add_images: true + global_average_pool: false + feature_rescale: 1.0 + view_pooler_args: + feature_aggregator_class_type: AngleWeightedReductionFeatureAggregator + view_sampler_args: + masked_sampling: false + sampling_mode: bilinear + feature_aggregator_IdentityFeatureAggregator_args: + exclude_target_view: true + exclude_target_view_mask_features: true + concatenate_output: true + feature_aggregator_ReductionFeatureAggregator_args: + exclude_target_view: true + exclude_target_view_mask_features: true + concatenate_output: true + reduction_functions: + - AVG + - STD + feature_aggregator_AngleWeightedReductionFeatureAggregator_args: + exclude_target_view: true + exclude_target_view_mask_features: true + concatenate_output: true + reduction_functions: + - AVG + - STD + weight_by_ray_angle_gamma: 1.0 + min_ray_angle_weight: 0.1 + feature_aggregator_AngleWeightedIdentityFeatureAggregator_args: + exclude_target_view: true + exclude_target_view_mask_features: true + concatenate_output: true + weight_by_ray_angle_gamma: 1.0 + min_ray_angle_weight: 0.1 + implicit_function_IdrFeatureField_args: + feature_vector_size: 3 + d_in: 3 + d_out: 1 + dims: + - 512 + - 512 + - 512 + - 512 + - 512 + - 512 + - 512 + - 512 + geometric_init: true + bias: 1.0 + skip_in: [] + weight_norm: true + n_harmonic_functions_xyz: 0 + pooled_feature_dim: 0 + encoding_dim: 0 + implicit_function_NeuralRadianceFieldImplicitFunction_args: + n_harmonic_functions_xyz: 10 + n_harmonic_functions_dir: 4 + n_hidden_neurons_dir: 128 + latent_dim: 0 + input_xyz: true + xyz_ray_dir_in_camera_coords: false + color_dim: 3 + transformer_dim_down_factor: 1.0 + n_hidden_neurons_xyz: 256 + n_layers_xyz: 8 + append_xyz: + - 5 + implicit_function_NeRFormerImplicitFunction_args: + n_harmonic_functions_xyz: 10 + n_harmonic_functions_dir: 4 + n_hidden_neurons_dir: 128 + latent_dim: 0 + input_xyz: true + xyz_ray_dir_in_camera_coords: false + color_dim: 3 + transformer_dim_down_factor: 2.0 + n_hidden_neurons_xyz: 80 + n_layers_xyz: 2 + append_xyz: + - 1 + implicit_function_SRNImplicitFunction_args: + raymarch_function_args: + n_harmonic_functions: 3 + n_hidden_units: 256 + n_layers: 2 + in_features: 3 + out_features: 256 + latent_dim: 0 + xyz_in_camera_coords: false + raymarch_function: null + pixel_generator_args: + n_harmonic_functions: 4 + n_hidden_units: 256 + n_hidden_units_color: 128 + n_layers: 2 + in_features: 256 + out_features: 3 + ray_dir_in_camera_coords: false + implicit_function_SRNHyperNetImplicitFunction_args: + hypernet_args: + n_harmonic_functions: 3 + n_hidden_units: 256 + n_layers: 2 + n_hidden_units_hypernet: 256 + n_layers_hypernet: 1 + in_features: 3 + out_features: 256 + latent_dim_hypernet: 0 + latent_dim: 0 + xyz_in_camera_coords: false + pixel_generator_args: + n_harmonic_functions: 4 + n_hidden_units: 256 + n_hidden_units_color: 128 + n_layers: 2 + in_features: 256 + out_features: 3 + ray_dir_in_camera_coords: false +solver_args: + breed: adam + weight_decay: 0.0 + lr_policy: multistep + lr: 0.0005 + gamma: 0.1 + momentum: 0.9 + betas: + - 0.9 + - 0.999 + milestones: [] + max_epochs: 1000 +data_source_args: + dataset_map_provider_class_type: ??? + data_loader_map_provider_class_type: SequenceDataLoaderMapProvider + dataset_map_provider_JsonIndexDatasetMapProvider_args: + category: ??? + task_str: singlesequence + dataset_root: '' + limit_to: -1 + limit_sequences_to: -1 + n_frames_per_sequence: -1 + test_on_train: false + load_point_clouds: false + mask_images: false + mask_depths: false + restrict_sequence_name: [] + test_restrict_sequence_id: -1 + assert_single_seq: false + only_test_set: false + aux_dataset_kwargs: + box_crop: true + box_crop_context: 0.3 + image_width: 800 + image_height: 800 + remove_empty_masks: true + path_manager: null + data_loader_map_provider_SequenceDataLoaderMapProvider_args: + batch_size: 1 + num_workers: 0 + dataset_len: 1000 + dataset_len_val: 1 + images_per_seq_options: + - 2 + sample_consecutive_frames: false + consecutive_frames_max_gap: 0 + consecutive_frames_max_gap_seconds: 0.1 +architecture: generic +detect_anomaly: false +eval_only: false +exp_dir: ./data/default_experiment/ +exp_idx: 0 +gpu_idx: 0 +metric_print_interval: 5 +resume: true +resume_epoch: -1 +seed: 0 +store_checkpoints: true +store_checkpoints_purge: 1 +test_interval: -1 +test_when_finished: false +validation_interval: 1 +visdom_env: '' +visdom_port: 8097 +visdom_server: http://127.0.0.1 +visualize_interval: 1000 +clip_grad: 0.0 +hydra: + run: + dir: . + output_subdir: null diff --git a/projects/implicitron_trainer/tests/test_experiment.py b/projects/implicitron_trainer/tests/test_experiment.py new file mode 100644 index 00000000..a7052aef --- /dev/null +++ b/projects/implicitron_trainer/tests/test_experiment.py @@ -0,0 +1,103 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# All rights reserved. +# +# This source code is licensed under the BSD-style license found in the +# LICENSE file in the root directory of this source tree. + +import logging +import os +import unittest +from pathlib import Path + +import experiment +import torch +from iopath.common.file_io import PathManager +from omegaconf import OmegaConf +from pytorch3d.implicitron.dataset.json_index_dataset_map_provider import ( + JsonIndexDatasetMapProvider, +) + + +def interactive_testing_requested() -> bool: + """ + Certain tests are only useful when run interactively, and so are not regularly run. + These are activated by this funciton returning True, which the user requests by + setting the environment variable `PYTORCH3D_INTERACTIVE_TESTING` to 1. + """ + return os.environ.get("PYTORCH3D_INTERACTIVE_TESTING", "") == "1" + + +DATA_DIR = Path(__file__).resolve().parent +DEBUG: bool = False + +# TODO: +# - sort out path_manager config. Here we monkeypatch to avoid +# the problem. +# - add enough files to skateboard_first_5 that this works on RE. +# - share common code with PyTorch3D tests? +# - deal with the temporary output files this test creates + + +def get_path_manager(silence_logs: bool = False) -> PathManager: + """ + Returns a path manager which can access manifold internally. + + Args: + silence_logs: Whether to reduce log output from iopath library. + """ + if silence_logs: + logging.getLogger("iopath.fb.manifold").setLevel(logging.CRITICAL) + logging.getLogger("iopath.common.file_io").setLevel(logging.CRITICAL) + + if os.environ.get("INSIDE_RE_WORKER", False): + raise ValueError("Cannot get to manifold from RE") + + path_manager = PathManager() + + if os.environ.get("FB_TEST", False): + from iopath.fb.manifold import ManifoldPathHandler + + path_manager.register_handler(ManifoldPathHandler()) + + return path_manager + + +def set_path_manager(self): + self.path_manager = get_path_manager() + + +class TestExperiment(unittest.TestCase): + def setUp(self): + self.maxDiff = None + JsonIndexDatasetMapProvider.__post_init__ = set_path_manager + + def test_from_defaults(self): + # Test making minimal changes to the dataclass defaults. + if not interactive_testing_requested(): + return + cfg = OmegaConf.structured(experiment.ExperimentConfig) + cfg.data_source_args.dataset_map_provider_class_type = ( + "JsonIndexDatasetMapProvider" + ) + dataset_args = ( + cfg.data_source_args.dataset_map_provider_JsonIndexDatasetMapProvider_args + ) + dataloader_args = ( + cfg.data_source_args.data_loader_map_provider_SequenceDataLoaderMapProvider_args + ) + dataset_args.category = "skateboard" + dataset_args.test_restrict_sequence_id = 0 + dataset_args.dataset_root = "manifold://co3d/tree/extracted" + dataset_args.limit_sequences_to = 5 + dataloader_args.dataset_len = 1 + cfg.solver_args.max_epochs = 2 + + device = torch.device("cuda:0") + experiment.run_training(cfg, device) + + def test_yaml_contents(self): + cfg = OmegaConf.structured(experiment.ExperimentConfig) + yaml = OmegaConf.to_yaml(cfg, sort_keys=False) + if DEBUG: + (DATA_DIR / "experiment.yaml").write_text(yaml) + self.assertEqual(yaml, (DATA_DIR / "experiment.yaml").read_text()) diff --git a/tests/common_testing.py b/tests/common_testing.py index cce23ea1..017e52ec 100644 --- a/tests/common_testing.py +++ b/tests/common_testing.py @@ -19,9 +19,9 @@ def interactive_testing_requested() -> bool: """ Certain tests are only useful when run interactively, and so are not regularly run. These are activated by this funciton returning True, which the user requests by - setting the environment variable `PYTORCH3D_INTERACTIVE_TESTING`. + setting the environment variable `PYTORCH3D_INTERACTIVE_TESTING` to 1. """ - return os.environ.get("PYTORCH3D_INTERACTIVE_TESTING", False) + return os.environ.get("PYTORCH3D_INTERACTIVE_TESTING", "") == "1" def get_tests_dir() -> Path: