pytorch

9441c7a9 - [JIT] add IR complexity tests (#34918)

[JIT] add IR complexity tests (#34918) Summary: Pull Request resolved: https://github.com/pytorch/pytorch/pull/34918 I'm going to set this up as a benchmarking test that runs internally in FB, but soliciting reviews externally first. I think that benchmarking complexity of our nn module & functional tests is useful because they are the building blocks of models, so they should be pretty representative of generic model complexity . This also separates out complexity benchmarking into tests that are easily debuggable given a regression, instead of a 50K node resnet graph. For each test, i am testing the profiled graph with consistent shapes, and I am testing - Number of If & loop statements - Number of non-tensor nodes (outputs don't include tensor) This is just a starting off point for testing IR complexity. Future plans could involve: - adding resnet, or other models in the model repo - benchmarking number of guards Current output: Functional tests: ``` ('Name', 'Ifs/Loops', 'non-tensor ops') ('conv1d', 0, 0) ('conv2d', 0, 0) ('conv3d', 0, 0) ('conv_transpose1d', 0, 0) ('conv_transpose2d', 0, 0) ('conv_transpose3d', 0, 0) ('conv_tbc', 0, 0) ('avg_pool1d', 0, 0) ('avg_pool2d', 0, 0) ('avg_pool3d', 0, 0) ('fractional_max_pool2d', 0, 3) ('max_pool1d', 0, 0) ('max_pool1d', 0, 0) ('max_pool2d', 0, 0) ('max_pool2d', 0, 0) ('max_pool3d', 0, 0) ('max_unpool1d', 0, 12) ('max_unpool2d', 0, 22) ('max_unpool3d', 0, 33) ('lp_pool1d', 0, 0) ('lp_pool2d', 0, 0) ('adaptive_max_pool1d', 0, 0) ('adaptive_max_pool2d', 0, 6) ('adaptive_max_pool3d', 0, 9) ('adaptive_avg_pool1d', 0, 0) ('adaptive_avg_pool2d', 0, 6) ('adaptive_avg_pool3d', 0, 9) ('dropout', 0, 0) ('alpha_dropout', 0, 0) ('dropout2d', 0, 0) ('dropout3d', 0, 0) ('feature_alpha_dropout', 0, 0) ('threshold', 0, 0) ('threshold', 0, 0) ('relu', 0, 0) ('relu', 0, 0) ('glu', 0, 0) ('hardtanh', 0, 0) ('hardtanh', 0, 0) ('relu6', 0, 0) ('relu6', 0, 0) ('elu', 0, 0) ('elu', 0, 0) ('selu', 0, 0) ('selu', 0, 0) ('celu', 0, 0) ('celu', 0, 0) ('leaky_relu', 0, 0) ('leaky_relu', 0, 0) ('rrelu', 0, 0) ('rrelu', 0, 0) ('hardshrink', 0, 0) ('tanhshrink', 0, 0) ('softsign', 0, 0) ('softplus', 0, 0) ('softmin', 0, 0) ('softmax', 0, 0) ('softmax', 0, 0) ('tanh', 0, 1) ('sigmoid', 0, 1) ('log_softmax', 0, 0) ('linear', 0, 0) ('linear', 0, 0) ('bilinear', 0, 0) ('embedding', 0, 0) ('embedding_bag', 0, 0) ('batch_norm', 0, 0) ('instance_norm', 1, 6) ('layer_norm', 0, 0) ('layer_norm', 0, 0) ('layer_norm', 0, 0) ('layer_norm', 0, 0) ('group_norm', 3, 53) ('local_response_norm', 0, 0) ('nll_loss', 1, 5) ('poisson_nll_loss', 0, 0) ('poisson_nll_loss', 0, 0) ('kl_div', 0, 1) ('cross_entropy', 1, 5) ('binary_cross_entropy_with_logits', 0, 0) ('smooth_l1_loss', 1, 1) ('l1_loss', 1, 1) ('mse_loss', 1, 1) ('smooth_l1_loss', 1, 1) ('l1_loss', 1, 1) ('mse_loss', 1, 1) ('margin_ranking_loss', 0, 0) ('hinge_embedding_loss', 0, 0) ('soft_margin_loss', 0, 0) ('multilabel_soft_margin_loss', 0, 1) ('cosine_embedding_loss', 0, 0) ('pixel_shuffle', 0, 0) ('affine_grid', 3, 14) ('pad', 0, 0) ('pairwise_distance', 0, 0) ('pdist', 0, 0) ('cosine_similarity', 0, 0) ('triplet_margin_loss', 0, 0) ('normalize', 0, 0) ('unfold', 0, 0) ('fold', 0, 0) ('grid_sample', 0, 1) ('gumbel_softmax', 0, 0) ('gumbel_softmax', 0, 0) ('multilabel_margin_loss', 0, 0) ('multi_margin_loss', 0, 0) ('binary_cross_entropy', 1, 5) ('binary_cross_entropy', 1, 5) ('ctc_loss', 0, 0) ('upsample', 13, 71) ('upsample', 13, 71) ('interpolate', 14, 71) ('interpolate', 13, 70) ('interpolate', 14, 71) ('interpolate', 14, 71) ('interpolate', 13, 70) ('interpolate', 14, 71) ('interpolate', 14, 71) ('interpolate', 13, 70) ('interpolate', 14, 71) ('interpolate', 14, 71) ('interpolate', 13, 70) ('interpolate', 14, 71) ('interpolate', 14, 60) ('interpolate', 13, 58) ('interpolate', 14, 60) ('interpolate', 14, 60) ('interpolate', 13, 58) ('interpolate', 14, 60) ('interpolate', 14, 60) ('interpolate', 13, 58) ('interpolate', 14, 60) ('interpolate', 13, 82) ('interpolate', 14, 82) ('interpolate', 14, 82) ('interpolate', 13, 82) ('interpolate', 14, 82) ('interpolate', 14, 82) ('interpolate', 13, 82) ('interpolate', 14, 82) ('interpolate', 14, 71) ('interpolate', 14, 71) ('interpolate', 15, 106) ('interpolate', 14, 73) ('interpolate', 15, 106) ('interpolate', 14, 73) ('interpolate', 15, 92) ('interpolate', 14, 60) ('interpolate', 15, 94) ('interpolate', 14, 62) ('interpolate', 15, 116) ('interpolate', 14, 82) ('interpolate', 15, 118) ('interpolate', 14, 84) ``` nn module tests: ``` ('Name', 'Ifs/Loops', 'non-tensor ops') ('test_nn_Linear', 0, 0) ('test_nn_Linear_no_bias', 0, 0) ('test_nn_Threshold_threshold_value', 0, 0) ('test_nn_Threshold_large_value', 0, 0) ('test_nn_ReLU', 0, 0) ('test_nn_ReLU6', 0, 0) ('test_nn_RReLU', 0, 0) ('test_nn_RReLU_with_up_down', 0, 0) ('test_nn_Hardtanh', 0, 0) ('test_nn_Sigmoid', 0, 0) ('test_nn_Tanh', 0, 0) ('test_nn_Flatten', 0, 0) ('test_nn_Softmax', 0, 0) ('test_nn_Softmax2d', 0, 0) ('test_nn_LogSoftmax', 0, 0) ('test_nn_LogSoftmax_multiparam', 0, 0) ('test_nn_ELU', 0, 0) ('test_nn_Hardshrink', 0, 0) ('test_nn_LeakyReLU', 0, 0) ('test_nn_LeakyReLU_with_negval', 0, 0) ('test_nn_LogSigmoid', 0, 0) ('test_nn_Softplus', 0, 0) ('test_nn_Softplus_beta', 0, 0) ('test_nn_Softplus_beta_threshold', 0, 0) ('test_nn_Softshrink', 0, 0) ('test_nn_Softshrink_lambda', 0, 0) ('test_nn_PReLU_1d', 0, 0) ('test_nn_PReLU_1d_multiparam', 0, 0) ('test_nn_PReLU_2d', 0, 0) ('test_nn_PReLU_2d_multiparam', 0, 0) ('test_nn_PReLU_3d', 0, 0) ('test_nn_PReLU_3d_multiparam', 0, 0) ('test_nn_Softsign', 0, 0) ('test_nn_Softmin', 0, 0) ('test_nn_Softmin_multidim', 0, 0) ('test_nn_Tanhshrink', 0, 0) ('test_nn_FractionalMaxPool2d_ratio', 0, 7) ('test_nn_FractionalMaxPool2d_size', 0, 0) ('test_nn_FractionalMaxPool3d_ratio', 0, 10) ('test_nn_FractionalMaxPool3d_size', 0, 0) ('test_nn_FractionalMaxPool3d_asymsize', 0, 0) ('test_nn_BatchNorm1d_affine', 2, 3) ('test_nn_BatchNorm1d_3d_input', 3, 9) ('test_nn_BatchNorm1d_affine_simple_average', 2, 5) ('test_nn_BatchNorm1d_not_affine', 2, 3) ('test_nn_BatchNorm1d_not_tracking_stats', 0, 0) ('test_nn_BatchNorm1d_3d_input_not_affine', 3, 9) ('test_nn_BatchNorm1d_zero_batch', 3, 9) ('test_nn_BatchNorm2d', 3, 13) ('test_nn_BatchNorm2d_2d_simple_average', 3, 15) ('test_nn_BatchNorm2d_momentum', 3, 13) ('test_nn_BatchNorm2d_not_affine', 3, 13) ('test_nn_BatchNorm2d_not_tracking_stats', 1, 10) ('test_nn_BatchNorm2d_zero_batch', 3, 13) ('test_nn_BatchNorm3d', 3, 17) ('test_nn_BatchNorm3d_3d_simple_average', 3, 19) ('test_nn_BatchNorm3d_momentum', 3, 17) ('test_nn_BatchNorm3d_not_affine', 3, 17) ('test_nn_BatchNorm3d_not_tracking_stats', 1, 14) ('test_nn_BatchNorm3d_zero_batch', 3, 17) ('test_nn_InstanceNorm1d', 1, 6) ('test_nn_InstanceNorm1d_tracking_stats', 1, 6) ('test_nn_InstanceNorm2d', 1, 10) ('test_nn_InstanceNorm2d_tracking_stats', 1, 10) ('test_nn_InstanceNorm3d', 1, 14) ('test_nn_InstanceNorm3d_tracking_stats', 1, 14) ('test_nn_LayerNorm_1d_elementwise_affine', 0, 0) ('test_nn_LayerNorm_1d_no_elementwise_affine', 0, 0) ('test_nn_LayerNorm_3d_elementwise_affine', 0, 0) ('test_nn_LayerNorm_3d_no_elementwise_affine', 0, 0) ('test_nn_LayerNorm_1d_empty_elementwise_affine', 0, 0) ('test_nn_GroupNorm_1d_affine', 3, 53) ('test_nn_GroupNorm_1d_no_affine_IN', 3, 53) ('test_nn_GroupNorm_1d_no_affine_LN', 3, 53) ('test_nn_GroupNorm_2d_affine', 3, 53) ('test_nn_GroupNorm_2d_no_affine_IN', 3, 53) ('test_nn_GroupNorm_2d_no_affine_LN', 3, 53) ('test_nn_Conv1d', 0, 0) ('test_nn_Conv1d_stride', 0, 0) ('test_nn_Conv1d_pad1', 0, 0) ('test_nn_Conv1d_pad2', 0, 0) ('test_nn_Conv1d_pad1size1', 0, 0) ('test_nn_Conv1d_pad2size1', 0, 0) ('test_nn_Conv1d_zero_batch', 0, 0) ('test_nn_Conv1d_dilated', 0, 0) ('test_nn_Conv1d_groups', 0, 0) ('test_nn_ConvTranspose1d', 0, 0) ('test_nn_ConvTranspose1d_no_bias', 0, 0) ('test_nn_ConvTranspose1d_dilated', 0, 0) ('test_nn_ConvTranspose1d_groups', 0, 0) ('test_nn_MaxPool1d', 0, 0) ('test_nn_MaxPool1d_stride', 0, 0) ('test_nn_Conv2d', 0, 0) ('test_nn_Conv2d_strided', 0, 0) ('test_nn_Conv2d_padding', 0, 0) ('test_nn_Conv2d_dilated', 0, 0) ('test_nn_Conv2d_no_bias', 0, 0) ('test_nn_Conv2d_zero_batch', 0, 0) ('test_nn_Conv2d_groups', 0, 0) ('test_nn_Conv2d_groups_thnn', 0, 0) ('test_nn_ConvTranspose2d', 0, 0) ('test_nn_ConvTranspose2d_dilated', 0, 0) ('test_nn_ConvTranspose2d_no_bias', 0, 0) ('test_nn_ConvTranspose2d_groups', 0, 0) ('test_nn_Conv2d_depthwise', 0, 0) ('test_nn_Conv2d_depthwise_with_multiplier', 0, 0) ('test_nn_Conv2d_depthwise_strided', 0, 0) ('test_nn_Conv2d_depthwise_padded', 0, 0) ('test_nn_Conv2d_depthwise_dilated', 0, 0) ('test_nn_MaxPool2d', 0, 0) ('test_nn_AvgPool1d', 0, 0) ('test_nn_AvgPool1d_stride', 0, 0) ('test_nn_AvgPool1d_stride_pad', 0, 0) ('test_nn_AvgPool2d', 0, 0) ('test_nn_AvgPool2d_stride', 0, 0) ('test_nn_AvgPool2d_stride_pad', 0, 0) ('test_nn_AvgPool2d_divisor', 0, 0) ('test_nn_AvgPool2d_divisor_stride', 0, 0) ('test_nn_AvgPool2d_divisor_stride_pad', 0, 0) ('test_nn_LPPool2d', 0, 0) ('test_nn_LPPool2d_norm', 0, 0) ('test_nn_LPPool1d_norm', 0, 0) ('test_nn_LPPool1d', 0, 0) ('test_nn_LocalResponseNorm_1d', 0, 0) ('test_nn_LocalResponseNorm_2d_uneven_pad', 0, 0) ('test_nn_LocalResponseNorm_3d_custom_params', 0, 0) ('test_nn_ReflectionPad1d', 0, 0) ('test_nn_ReflectionPad2d', 0, 0) ('test_nn_ReplicationPad1d', 0, 0) ('test_nn_ReplicationPad2d', 0, 0) ('test_nn_ZeroPad2d', 0, 0) ('test_nn_ZeroPad2d_negative_dims', 0, 0) ('test_nn_ConstantPad1d', 0, 0) ('test_nn_ConstantPad2d', 0, 0) ('test_nn_ConstantPad3d', 0, 0) ('test_nn_Conv3d', 0, 0) ('test_nn_Conv3d_no_bias', 0, 0) ('test_nn_Conv3d_stride', 0, 0) ('test_nn_Conv3d_stride_padding', 0, 0) ('test_nn_Conv3d_zero_batch', 0, 0) ('test_nn_Conv3d_groups', 0, 0) ('test_nn_Conv3d_dilated', 0, 0) ('test_nn_Conv3d_dilated_strided', 0, 0) ('test_nn_ConvTranspose3d', 0, 0) ('test_nn_ConvTranspose3d_dilated', 0, 0) ('test_nn_MaxPool3d', 0, 0) ('test_nn_MaxPool3d_stride', 0, 0) ('test_nn_MaxPool3d_stride_padding', 0, 0) ('test_nn_AvgPool3d', 0, 0) ('test_nn_AvgPool3d_stride', 0, 0) ('test_nn_AvgPool3d_stride_pad', 0, 0) ('test_nn_AvgPool3d_stride_pad_gpu_fixedkw_output', 0, 0) ('test_nn_AvgPool3d_stride_pad_gpu_general_output', 0, 0) ('test_nn_AvgPool3d_stride1_pad0_gpu_input', 0, 0) ('test_nn_AvgPool3d_stride_pad_gpu_input_nooverlap', 0, 0) ('test_nn_AvgPool3d_divisor', 0, 0) ('test_nn_AvgPool3d_divisor_stride', 0, 0) ('test_nn_AvgPool3d_divisor_stride_pad', 0, 0) ('test_nn_AvgPool3d_divisor_stride_pad_gpu_fixedkw_output', 0, 0) ('test_nn_AvgPool3d_divisor_stride_pad_gpu_general_output', 0, 0) ('test_nn_AvgPool3d_divisor_stride1_pad0_gpu_input', 0, 0) ('test_nn_AvgPool3d_divisor_stride_pad_gpu_input_nooverlap', 0, 0) ('test_nn_ReplicationPad3d', 0, 0) ('test_nn_Embedding', 0, 0) ('test_nn_EmbeddingBag_mean', 0, 2) ('test_nn_EmbeddingBag_sum', 0, 2) ('test_nn_EmbeddingBag_max', 0, 2) ('test_nn_EmbeddingBag_sparse', 0, 2) ('test_nn_Embedding_sparse', 0, 0) ('test_nn_PixelShuffle', 0, 0) ('test_nn_AdaptiveMaxPool1d', 0, 0) ('test_nn_AdaptiveMaxPool2d_single', 0, 6) ('test_nn_AdaptiveMaxPool2d_tuple', 0, 6) ('test_nn_AdaptiveMaxPool3d_single', 0, 9) ('test_nn_AdaptiveMaxPool3d_tuple', 0, 9) ('test_nn_AdaptiveMaxPool3d_single_nonatomic', 0, 9) ('test_nn_AdaptiveMaxPool3d_tuple_nonatomic', 0, 9) ('test_nn_AdaptiveAvgPool1d', 0, 0) ('test_nn_AdaptiveAvgPool1d_one_output', 0, 0) ('test_nn_AdaptiveAvgPool2d_single', 0, 6) ('test_nn_AdaptiveAvgPool2d_single_1x1output', 0, 6) ('test_nn_AdaptiveAvgPool2d_tuple', 0, 6) ('test_nn_AdaptiveAvgPool3d_single', 0, 9) ('test_nn_AdaptiveAvgPool3d_tuple', 0, 9) ('test_nn_SELU', 0, 0) ('test_nn_SELU_scalar', 0, 0) ('test_nn_CELU', 0, 0) ('test_nn_CELU_scalar', 0, 0) ('test_nn_GLU', 0, 0) ('test_nn_GLU_dim', 0, 0) ('test_nn_GELU_scalar', 0, 0) ('test_nn_GELU', 0, 0) ('test_nn_Unfold', 0, 0) ('test_nn_Fold', 0, 0) ('test_nn_Unfold_int_input', 0, 0) ('test_nn_Fold_int_input', 0, 0) ('test_nn_Threshold_threshold_value_scalar', 0, 0) ('test_nn_ReLU_scalar', 0, 0) ('test_nn_ReLU6_scalar', 0, 0) ('test_nn_RReLU_with_up_down_scalar', 0, 0) ('test_nn_Hardtanh_scalar', 0, 0) ('test_nn_Sigmoid_scalar', 0, 0) ('test_nn_Tanh_scalar', 0, 0) ('test_nn_Softmax_scalar', 0, 0) ('test_nn_LogSoftmax_multiparam_scalar', 0, 0) ('test_nn_ELU_scalar', 0, 0) ('test_nn_Hardshrink_scalar', 0, 0) ('test_nn_LeakyReLU_with_negval_scalar', 0, 0) ('test_nn_LogSigmoid_scalar', 0, 0) ('test_nn_Softplus_beta_threshold_scalar', 0, 0) ('test_nn_Softshrink_lambda_scalar', 0, 0) ('test_nn_PReLU_scalar', 0, 0) ('test_nn_Softsign_scalar', 0, 0) ('test_nn_Softmin_scalar', 0, 0) ('test_nn_Tanhshrink_scalar', 0, 0) ('test_nn_Conv1d_reflect_stride2_pad2', 3, 14) ('test_nn_Conv2d_reflect_stride2_pad2', 3, 14) ('test_nn_Conv1d_circular_stride2_pad2', 5, 31) ('test_nn_Conv2d_circular_stride2_pad2', 5, 31) ('test_nn_Conv3d_circular_stride2_pad2', 5, 31) ('test_nn_Conv1d_replicate_stride2_pad2', 3, 14) ('test_nn_Conv2d_replicate_stride2_pad2', 3, 14) ('test_nn_Conv3d_replicate_stride2_pad2', 3, 14) ('test_nn_Conv1d_zeros_stride2_pad2', 0, 0) ('test_nn_Conv2d_zeros_stride2_pad2', 0, 0) ('test_nn_Conv3d_zeros_stride2_pad2', 0, 0) ('test_nn_Bilinear', 0, 0) ('test_nn_RNNCell', 3, 14) ('test_nn_LSTMCell', 5, 22) ('test_nn_GRUCell', 3, 14) ('test_nn_MultiheadAttention', 40, 160) ('test_nn_Transformer', 128, 499) ``` Test Plan: Imported from OSS Differential Revision: D20539336 Pulled By: eellison fbshipit-source-id: 14ac00a7b2b029b9e57f6131dd45426b0101941a