Index

_

__call__() (xffl.custom.types.FileLike method) (xffl.custom.types.FolderLike method) (xffl.custom.types.PathLike method) __str__() (xffl.distributed.distributed_state.DistributedState method) _add_arguments_option() (in module xffl.cli.parser) _add_common_project_options() (in module xffl.cli.parser) _all_reduce() (in module xffl.distributed.utils) _all_reduce_and_broadcast() (in module xffl.distributed.utils) _configure_facility() (in module xffl.cli.config) _flavour (xffl.custom.types.FileLike attribute) (xffl.custom.types.FolderLike attribute) (xffl.custom.types.PathLike attribute) _get_communicating_processes() (xffl.distributed.distributed_state.DistributedState method) _get_current_device() (in module xffl.distributed.distributed)

_get_global_fsdp_mesh() (xffl.distributed.distributed_state.DistributedState method) _get_init_device() (in module xffl.distributed.distributed) _get_model_info() (in module xffl.cli.config) _get_training_info() (in module xffl.cli.config) _is_broadcast_necessary() (in module xffl.distributed.utils) _partial_hsdp_setup() (xffl.distributed.distributed_state.DistributedState method) _partial_hsdp_setup_manual() (xffl.distributed.distributed_state.DistributedState method) _set_asymmetric_federated_scaling() (xffl.distributed.distributed_state.DistributedState method) _set_global_hsdp_mesh() (xffl.distributed.distributed_state.DistributedState method) _set_rank_role() (xffl.distributed.distributed_state.DistributedState method) _set_symmetric_federated_scaling() (xffl.distributed.distributed_state.DistributedState method) _setup_devices() (in module xffl.distributed.distributed) _setup_streams() (in module xffl.distributed.utils) _write_output_files() (in module xffl.cli.config)

A

add_deployment() (xffl.workflow.templates.streamflow.StreamFlowFile method) add_inputs() (xffl.workflow.templates.cwl.AggregateStep method) (xffl.workflow.templates.cwl.CWLConfig method) (xffl.workflow.templates.cwl.MainWorkflow method) (xffl.workflow.templates.cwl.RoundWorkflow method) (xffl.workflow.templates.cwl.TrainingStep method) (xffl.workflow.templates.cwl.Workflow method) (xffl.workflow.templates.streamflow.StreamFlowFile method)

add_root_step() (xffl.workflow.templates.streamflow.StreamFlowFile method) add_training_step() (xffl.workflow.templates.streamflow.StreamFlowFile method) AggregateStep (class in xffl.workflow.templates.cwl) ArgumentParser (class in xffl.custom.parser)

B

backend (xffl.distributed.distributed_state.DistributedState attribute) benchmark_aggregation() (in module xffl.distributed.aggregation) BLUE (xffl.custom.formatter.CustomFormatter attribute) BOLD_RED (xffl.custom.formatter.CustomFormatter attribute) broadcast (xffl.distributed.utils.Strategy attribute) bucket_coalesced() (in module xffl.distributed.aggregation) bucket_coalesced_() (in module xffl.distributed.aggregation)

bucket_flatten() (in module xffl.distributed.aggregation) bucket_flatten_() (in module xffl.distributed.aggregation) bucket_optimized_coalesced() (in module xffl.distributed.aggregation) bucket_optimized_coalesced_() (in module xffl.distributed.aggregation) bucket_optimized_flatten() (in module xffl.distributed.aggregation) bucket_optimized_flatten_() (in module xffl.distributed.aggregation) build_parser() (in module xffl.cli.parser)

C

check_and_create_dir() (in module xffl.cli.utils) check_default_value() (in module xffl.cli.utils) check_input() (in module xffl.utils.utils) CleanMc4It (class in xffl.custom.datasets) (class in xffl.custom.datasets.clean_mc4_it) cleanup_distributed_process_group() (in module xffl.distributed.distributed) COMMANDS (in module xffl.cli.cli) config() (in module xffl.cli.config) content (xffl.workflow.config.YamlConfig attribute)

create_binding() (xffl.workflow.templates.streamflow.StreamFlowFile static method) create_device_mesh() (in module xffl.distributed.distributed_state) create_fsdp_model() (in module xffl.learning.modelling) create_process_group() (xffl.distributed.distributed_state.DistributedState method) current_device (xffl.distributed.distributed_state.DistributedState attribute) CustomFormatter (class in xffl.custom.formatter) cwl (xffl.workflow.templates.cwl.Workflow attribute) CWL_TYPE_MAPPING (in module xffl.workflow.utils) CWLConfig (class in xffl.workflow.templates.cwl)

D

DatasetInfo (class in xffl.custom.datasets) (class in xffl.custom.datasets.dataset_info) DATASETS (in module xffl.custom.datasets) DATE_FORMAT (xffl.custom.formatter.CustomFormatter attribute) decoder_layer (xffl.custom.models.Llama attribute) (xffl.custom.models.llama.Llama attribute) (xffl.custom.models.Mixtral attribute) (xffl.custom.models.mixtral.Mixtral attribute) (xffl.custom.models.model_info.ModelInfo attribute) (xffl.custom.models.ModelInfo attribute)

DEFAULT_FORMAT (xffl.custom.formatter.CustomFormatter attribute) DEFAULT_xFFL_DIR (in module xffl.utils.constants) deployments (xffl.workflow.templates.streamflow.StreamFlowFile attribute) device_type (xffl.distributed.distributed_state.DistributedState attribute) dispatch_command() (in module xffl.cli.cli) distributed_training() (in module xffl.learning.processing) DistributedState (class in xffl.distributed.distributed_state)

E

exclude (xffl.custom.formatter.ExcludeLoggerFilter attribute) EXCLUDED_LOGGERS (in module xffl.utils.logging)

ExcludeLoggerFilter (class in xffl.custom.formatter) exec() (in module xffl.cli.exec) expand_paths_in_args() (in module xffl.cli.utils)

F

federated_group (xffl.distributed.distributed_state.DistributedState attribute) federated_local_rank (xffl.distributed.distributed_state.DistributedState attribute) federated_local_size (xffl.distributed.distributed_state.DistributedState attribute) federated_rank (xffl.distributed.distributed_state.DistributedState attribute) federated_world_size (xffl.distributed.distributed_state.DistributedState attribute) federation (xffl.distributed.distributed_state.DistributedState attribute) FileLike (class in xffl.custom.types)

filter() (xffl.custom.formatter.ExcludeLoggerFilter method) fmt (xffl.custom.formatter.CustomFormatter attribute) FolderLike (class in xffl.custom.types) format() (xffl.custom.formatter.CustomFormatter method) FORMATS (xffl.custom.formatter.CustomFormatter attribute) from_args_to_cwl() (in module xffl.workflow.utils) fsdp_evaluation() (in module xffl.learning.processing) fsdp_mesh (xffl.distributed.distributed_state.DistributedState attribute)

G

get_appropriate_sharding_strategy() (in module xffl.distributed.distributed) get_available_position() (xffl.workflow.templates.cwl.TrainingStep method) get_cells_ids() (in module xffl.distributed.networking) get_default_content() (xffl.workflow.config.YamlConfig class method) (xffl.workflow.templates.cwl.CWLConfig class method) (xffl.workflow.templates.streamflow.StreamFlowFile class method) get_default_definition() (xffl.workflow.templates.cwl.AggregateStep class method) (xffl.workflow.templates.cwl.MainWorkflow class method) (xffl.workflow.templates.cwl.RoundWorkflow class method) (xffl.workflow.templates.cwl.TrainingStep class method) (xffl.workflow.templates.cwl.Workflow class method)

get_default_handler() (in module xffl.utils.logging) get_default_nccl_process_group_options() (in module xffl.utils.utils) get_facilitator_path() (in module xffl.cli.utils) get_model_size() (in module xffl.learning.utils) get_model_size_in_bits() (in module xffl.learning.utils) get_param_flag() (in module xffl.utils.utils) get_param_name() (in module xffl.utils.utils) get_timeout() (in module xffl.utils.utils) get_training_step() (xffl.workflow.templates.cwl.RoundWorkflow class method) GREY (xffl.custom.formatter.CustomFormatter attribute)

H

hsdp_mesh (xffl.distributed.distributed_state.DistributedState attribute)

I

import_from_path() (in module xffl.workflow.utils) init_device (xffl.distributed.distributed_state.DistributedState attribute) init_distributed_process_group() (in module xffl.distributed.distributed) is_federated_scaling_setup() (xffl.distributed.distributed_state.DistributedState method)

is_fsdp_setup() (xffl.distributed.distributed_state.DistributedState method) is_hsdp_setup() (xffl.distributed.distributed_state.DistributedState method) is_node_setup() (xffl.distributed.distributed_state.DistributedState method) is_sender (xffl.distributed.distributed_state.DistributedState attribute)

L

layer_by_layer() (in module xffl.distributed.aggregation) layer_by_layer_() (in module xffl.distributed.aggregation) layer_by_layer_optimized() (in module xffl.distributed.aggregation) layer_by_layer_optimized_() (in module xffl.distributed.aggregation) Llama (class in xffl.custom.models) (class in xffl.custom.models.llama) load_datasets_from_disk() (in module xffl.learning.data) logger (in module xffl.cli.cli) (in module xffl.cli.config) (in module xffl.cli.exec) (in module xffl.cli.run) (in module xffl.cli.utils) (in module xffl.distributed.aggregation) (in module xffl.distributed.distributed) (in module xffl.distributed.distributed_state) (in module xffl.distributed.utils) (in module xffl.learning.data) (in module xffl.learning.modelling) (in module xffl.learning.processing) (in module xffl.learning.utils) (in module xffl.utils.utils) (in module xffl.workflow.streamflow) (in module xffl.workflow.templates.cwl) (in module xffl.workflow.utils)

M

main() (in module xffl.cli.cli) (in module xffl.cli.config) (in module xffl.cli.exec) (in module xffl.cli.run) (in module xffl.workflow.templates.aggregation_application) MainWorkflow (class in xffl.workflow.templates.cwl) mapping (xffl.distributed.utils.Strategy attribute) master_addr (xffl.distributed.distributed_state.DistributedState attribute) master_port (xffl.distributed.distributed_state.DistributedState attribute) meta_initialization (xffl.distributed.distributed_state.DistributedState attribute) Mixtral (class in xffl.custom.models) (class in xffl.custom.models.mixtral) model_type (xffl.custom.models.Llama attribute) (xffl.custom.models.llama.Llama attribute) (xffl.custom.models.Mixtral attribute) (xffl.custom.models.mixtral.Mixtral attribute) (xffl.custom.models.model_info.ModelInfo attribute) (xffl.custom.models.ModelInfo attribute) ModelInfo (class in xffl.custom.models) (class in xffl.custom.models.model_info) MODELS (in module xffl.custom.models) module xffl xffl.cli xffl.cli.cli xffl.cli.config xffl.cli.exec xffl.cli.parser xffl.cli.run xffl.cli.utils xffl.custom xffl.custom.datasets xffl.custom.datasets.clean_mc4_it xffl.custom.datasets.dataset_info xffl.custom.formatter xffl.custom.models xffl.custom.models.llama xffl.custom.models.mixtral xffl.custom.models.model_info xffl.custom.parser xffl.custom.types xffl.distributed xffl.distributed.aggregation xffl.distributed.distributed xffl.distributed.distributed_state xffl.distributed.networking xffl.distributed.utils xffl.learning xffl.learning.data xffl.learning.modelling xffl.learning.processing xffl.learning.utils xffl.utils xffl.utils.constants xffl.utils.logging xffl.utils.utils xffl.workflow xffl.workflow.config xffl.workflow.streamflow xffl.workflow.templates xffl.workflow.templates.aggregation_application xffl.workflow.templates.cwl xffl.workflow.templates.streamflow xffl.workflow.utils

N

node_local_rank (xffl.distributed.distributed_state.DistributedState attribute) node_local_size (xffl.distributed.distributed_state.DistributedState attribute)

node_rank (xffl.distributed.distributed_state.DistributedState attribute) node_world_size (xffl.distributed.distributed_state.DistributedState attribute)

P

parser (in module xffl.workflow.templates.aggregation_application) path (xffl.custom.datasets.clean_mc4_it.CleanMc4It attribute) (xffl.custom.datasets.CleanMc4It attribute) (xffl.custom.datasets.dataset_info.DatasetInfo attribute) (xffl.custom.datasets.DatasetInfo attribute) (xffl.custom.models.Llama attribute) (xffl.custom.models.llama.Llama attribute) (xffl.custom.models.Mixtral attribute) (xffl.custom.models.mixtral.Mixtral attribute) (xffl.custom.models.model_info.ModelInfo attribute) (xffl.custom.models.ModelInfo attribute)

PathLike (class in xffl.custom.types) preload() (in module xffl.learning.utils)

R

rank (xffl.distributed.distributed_state.DistributedState attribute) receive_from (xffl.distributed.distributed_state.DistributedState attribute) RED (xffl.custom.formatter.CustomFormatter attribute) reduce_op (xffl.distributed.utils.Strategy attribute) replica_group (xffl.distributed.distributed_state.DistributedState attribute) replica_local_rank (xffl.distributed.distributed_state.DistributedState attribute) replica_local_size (xffl.distributed.distributed_state.DistributedState attribute)

replica_rank (xffl.distributed.distributed_state.DistributedState attribute) replica_world_size (xffl.distributed.distributed_state.DistributedState attribute) RESET (xffl.custom.formatter.CustomFormatter attribute) resolve_path() (in module xffl.utils.utils) RoundWorkflow (class in xffl.workflow.templates.cwl) run() (in module xffl.cli.cli) (in module xffl.cli.run) run_streamflow() (in module xffl.workflow.streamflow)

S

save() (xffl.workflow.config.YamlConfig method) (xffl.workflow.templates.cwl.Workflow method) save_fsdp_model() (in module xffl.learning.modelling) seed_dataloader_worker() (in module xffl.learning.utils) set_activation_checkpointing() (in module xffl.learning.utils) set_deterministic_execution() (in module xffl.learning.utils) set_exec_device() (xffl.distributed.distributed_state.DistributedState method) set_federated_scaling() (xffl.distributed.distributed_state.DistributedState method) set_fsdp() (xffl.distributed.distributed_state.DistributedState method) set_global() (xffl.distributed.distributed_state.DistributedState method) set_hsdp() (xffl.distributed.distributed_state.DistributedState method) set_init_device() (xffl.distributed.distributed_state.DistributedState method) set_node() (xffl.distributed.distributed_state.DistributedState method)

set_nondeterministic_execution() (in module xffl.learning.utils) setup_distributed_process_group() (in module xffl.distributed.distributed) setup_env() (in module xffl.cli.utils) setup_execution_env() (in module xffl.cli.exec) setup_logging() (in module xffl.utils.logging) splits (xffl.custom.datasets.clean_mc4_it.CleanMc4It attribute) (xffl.custom.datasets.CleanMc4It attribute) (xffl.custom.datasets.dataset_info.DatasetInfo attribute) (xffl.custom.datasets.DatasetInfo attribute) src (xffl.distributed.utils.Strategy attribute) step_bindings (xffl.workflow.templates.streamflow.StreamFlowFile attribute) Strategy (class in xffl.distributed.utils) StreamFlowFile (class in xffl.workflow.templates.streamflow) streams (xffl.distributed.distributed_state.DistributedState attribute)

T

TrainingStep (class in xffl.workflow.templates.cwl)

U

unset_federated_scaling() (xffl.distributed.distributed_state.DistributedState method) unset_hsdp() (xffl.distributed.distributed_state.DistributedState method)

update_merge_step() (xffl.workflow.templates.cwl.RoundWorkflow method) use_contiguous_memory (xffl.distributed.utils.Strategy attribute)

V

VERSION (in module xffl.utils.constants)

W

Workflow (class in xffl.workflow.templates.cwl) world_size (xffl.distributed.distributed_state.DistributedState attribute) wrapping_policy (xffl.custom.models.Llama attribute) (xffl.custom.models.llama.Llama attribute) (xffl.custom.models.Mixtral attribute) (xffl.custom.models.mixtral.Mixtral attribute) (xffl.custom.models.model_info.ModelInfo attribute) (xffl.custom.models.ModelInfo attribute)

X

xffl module xffl.cli module xffl.cli.cli module xffl.cli.config module xffl.cli.exec module xffl.cli.parser module xffl.cli.run module xffl.cli.utils module xffl.custom module xffl.custom.datasets module xffl.custom.datasets.clean_mc4_it module xffl.custom.datasets.dataset_info module xffl.custom.formatter module xffl.custom.models module xffl.custom.models.llama module xffl.custom.models.mixtral module xffl.custom.models.model_info module xffl.custom.parser module xffl.custom.types module xffl.distributed module xffl.distributed.aggregation module

xffl.distributed.distributed module xffl.distributed.distributed_state module xffl.distributed.networking module xffl.distributed.utils module xffl.learning module xffl.learning.data module xffl.learning.modelling module xffl.learning.processing module xffl.learning.utils module xffl.utils module xffl.utils.constants module xffl.utils.logging module xffl.utils.utils module xffl.workflow module xffl.workflow.config module xffl.workflow.streamflow module xffl.workflow.templates module xffl.workflow.templates.aggregation_application module xffl.workflow.templates.cwl module xffl.workflow.templates.streamflow module xffl.workflow.utils module

Y

YamlConfig (class in xffl.workflow.config)

YELLOW (xffl.custom.formatter.CustomFormatter attribute)