all_gather.py
Commit
2 years ago
Author
- Files81
- benchmarks
- communication
- all_reduce.py
- all_to_all.py
- broadcast.py
- constants.py
- pt2pt.py
- utils.py
- inference
- bert-bench.py
- gpt-bench.py
- deepspeed
- comm
- comm.py
- torch.py
- inference
- engine.py
- model_implementations
- diffusers
- unet.py
- vae.py
- features
- __init__.py
- cuda_graph.py
- transformers
- clip_encoder.py
- module_inject
- auto_tp.py
- containers
- base.py
- base_moe.py
- unet.py
- vae.py
- policy.py
- runtime
- fp16
- loss_scaler.py
- zero
- stage3.py
- stage_1_and_2.py
- docs/_tutorials
cifar-10.md
- tests
- accelerator
ds_config.json- test_ds_init.py
- benchmarks
- flatten_bench.py
- unflatten_bench.py
- onebit
- test_mpi_backend.py
- test_mpi_perf.py
- test_nccl_backend.py
- test_nccl_perf.py
- perf
- adam_test1.py
- small_model_debugging
- test.py
- unit
- alexnet_model.py
- comm
- test_dist.py
- common.py
- compression
- test_compression.py
- inference
- test_inference.py
- test_model_profiling.py
- megatron_model.py
- model_parallelism
- test_configurable_parallel_mp.py
- test_configurable_parallel_pp.py
- modeling.py
- modelingpreln.py
- ops
- accelerators
- test_accelerator_backward.py
- test_accelerator_forward.py
- adagrad
- test_cpu_adagrad.py
- adam
- test_cpu_adam.py
- aio
- test_aio.py
- quantizer
- test_dequantize.py
- test_fake_quantization.py
- test_quantize.py
- sparse_attention
- test_sparse_attention.py
- spatial
- test_nhwc_bias_add.py
- transformer/inference
- test_bias_add.py
- test_bias_geglu.py
- test_bias_gelu.py
- test_bias_relu.py
- test_layer_norm.py
- test_moe_res_matmult.py
- test_residual_add.py
- pipe
- test_pipe_module.py
- runtime
- activation_checkpointing
- test_activation_checkpointing.py
- comm
- test_coalesced_collectives.py
- half_precision
- onebit
- test_onebit.py
- test_fp16.py
- pipe
- test_topology.py
- test_autocast.py
- test_data.py
- test_data_efficiency.py
- test_ds_config_dict.py
- test_runtime_utils.py
- utils
- test_partition.py
- zero
- test_zero.py
- test_zero_context_ancestry.py
- simple_model.py
- utils
- test_init_on_device.py