Buckets:

hf-doc-build
/

doc-dev

Files
xet
hf-doc-build/doc-dev / diffusers /pr_14972 /en /optimization /tensor_parallelism.html
HuggingFaceDocBuilder
about 7 hours ago
download
raw
38.1 kB
<meta charset="utf-8" /><meta name="hf:doc:metadata" content="{&quot;title&quot;:&quot;Tensor parallelism&quot;,&quot;local&quot;:&quot;tensor-parallelism&quot;,&quot;sections&quot;:[{&quot;title&quot;:&quot;Load a sharded model&quot;,&quot;local&quot;:&quot;load-a-sharded-model&quot;,&quot;sections&quot;:[],&quot;depth&quot;:2},{&quot;title&quot;:&quot;Loading performance&quot;,&quot;local&quot;:&quot;loading-performance&quot;,&quot;sections&quot;:[],&quot;depth&quot;:2},{&quot;title&quot;:&quot;Limitations&quot;,&quot;local&quot;:&quot;limitations&quot;,&quot;sections&quot;:[],&quot;depth&quot;:2},{&quot;title&quot;:&quot;Add tensor parallelism to a model&quot;,&quot;local&quot;:&quot;add-tensor-parallelism-to-a-model&quot;,&quot;sections&quot;:[{&quot;title&quot;:&quot;Colwise and rowwise&quot;,&quot;local&quot;:&quot;colwise-and-rowwise&quot;,&quot;sections&quot;:[],&quot;depth&quot;:3},{&quot;title&quot;:&quot;Fused projections&quot;,&quot;local&quot;:&quot;fused-projections&quot;,&quot;sections&quot;:[],&quot;depth&quot;:3},{&quot;title&quot;:&quot;What to leave out&quot;,&quot;local&quot;:&quot;what-to-leave-out&quot;,&quot;sections&quot;:[],&quot;depth&quot;:3},{&quot;title&quot;:&quot;Constraints and verification&quot;,&quot;local&quot;:&quot;constraints-and-verification&quot;,&quot;sections&quot;:[],&quot;depth&quot;:3}],&quot;depth&quot;:2}],&quot;depth&quot;:1}"/>
		<link href="/docs/diffusers/main/en/_app/immutable/entry/start.0S0_iEoG.js" rel="modulepreload">
		<link href="/docs/diffusers/main/en/_app/immutable/chunks/I9eooN77.js" rel="modulepreload">
		<link href="/docs/diffusers/main/en/_app/immutable/chunks/Bupkjr9f.js" rel="modulepreload">
		<link href="/docs/diffusers/main/en/_app/immutable/entry/app.CPdOxCbw.js" rel="modulepreload">
		<link href="/docs/diffusers/main/en/_app/immutable/chunks/CyvZ3FS5.js" rel="modulepreload">
		<link href="/docs/diffusers/main/en/_app/immutable/chunks/l0QnnOED.js" rel="modulepreload">
		<link href="/docs/diffusers/main/en/_app/immutable/chunks/DnVIJfC5.js" rel="modulepreload">
		<link href="/docs/diffusers/main/en/_app/immutable/chunks/DsnmJJEf.js" rel="modulepreload">
		<link href="/docs/diffusers/main/en/_app/immutable/nodes/0.DteTvmQZ.js" rel="modulepreload">
		<link href="/docs/diffusers/main/en/_app/immutable/chunks/CoSrdQax.js" rel="modulepreload">
		<link href="/docs/diffusers/main/en/_app/immutable/nodes/347.DnqzUbRP.js" rel="modulepreload">
		<link href="/docs/diffusers/main/en/_app/immutable/chunks/DSPDl1gz.js" rel="modulepreload">
		<!--17gkrup--><meta name="hf:doc:metadata" content="{&quot;title&quot;:&quot;Tensor parallelism&quot;,&quot;local&quot;:&quot;tensor-parallelism&quot;,&quot;sections&quot;:[{&quot;title&quot;:&quot;Load a sharded model&quot;,&quot;local&quot;:&quot;load-a-sharded-model&quot;,&quot;sections&quot;:[],&quot;depth&quot;:2},{&quot;title&quot;:&quot;Loading performance&quot;,&quot;local&quot;:&quot;loading-performance&quot;,&quot;sections&quot;:[],&quot;depth&quot;:2},{&quot;title&quot;:&quot;Limitations&quot;,&quot;local&quot;:&quot;limitations&quot;,&quot;sections&quot;:[],&quot;depth&quot;:2},{&quot;title&quot;:&quot;Add tensor parallelism to a model&quot;,&quot;local&quot;:&quot;add-tensor-parallelism-to-a-model&quot;,&quot;sections&quot;:[{&quot;title&quot;:&quot;Colwise and rowwise&quot;,&quot;local&quot;:&quot;colwise-and-rowwise&quot;,&quot;sections&quot;:[],&quot;depth&quot;:3},{&quot;title&quot;:&quot;Fused projections&quot;,&quot;local&quot;:&quot;fused-projections&quot;,&quot;sections&quot;:[],&quot;depth&quot;:3},{&quot;title&quot;:&quot;What to leave out&quot;,&quot;local&quot;:&quot;what-to-leave-out&quot;,&quot;sections&quot;:[],&quot;depth&quot;:3},{&quot;title&quot;:&quot;Constraints and verification&quot;,&quot;local&quot;:&quot;constraints-and-verification&quot;,&quot;sections&quot;:[],&quot;depth&quot;:3}],&quot;depth&quot;:2}],&quot;depth&quot;:1}"/><!---->
		<link href="/docs/diffusers/main/en/_app/immutable/assets/0.tn0RQdqM.css" rel="modulepreload"> <!--[--><!--[0--><!--[--><!--[0--><!--[--><p></p> <div class="items-center shrink-0 min-w-[100px] max-sm:min-w-[50px] justify-end ml-auto flex" style="float: inline-end; margin-inline-start: 10px; display: inline-flex; position: relative; z-index: 10;"><div dir="ltr" class="inline-flex rounded-md max-sm:rounded-sm"><button class="inline-flex items-center gap-1 h-7 max-sm:h-7 px-2 max-sm:px-1.5 text-sm font-medium text-gray-800 border border-r-0 rounded-l-md max-sm:rounded-l-sm border-gray-200 bg-white hover:shadow-inner dark:border-gray-850 dark:bg-gray-950 dark:text-gray-200 dark:hover:bg-gray-800" aria-live="polite"><span class="inline-flex items-center justify-center rounded-md p-0.5 max-sm:p-0 hover:text-gray-800 dark:hover:text-gray-200"><svg class="sm:size-3.5 size-3" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg><!----></span> <span dir="auto">Copy page</span></button> <button class="inline-flex items-center justify-center w-6 max-sm:w-5 h-7 max-sm:h-7 disabled:pointer-events-none text-sm text-gray-500 hover:text-gray-700 dark:hover:text-white rounded-r-md max-sm:rounded-r-sm border border-l transition border-gray-200 bg-white hover:shadow-inner dark:border-gray-850 dark:bg-gray-950 dark:text-gray-200 dark:hover:bg-gray-800" aria-haspopup="menu" aria-expanded="false" aria-label="Open copy menu"><svg class="transition-transform text-gray-400 overflow-visible sm:size-3.5 size-3 rotate-0" width="1em" height="1em" viewBox="0 0 12 7" fill="none" xmlns="http://www.w3.org/2000/svg"><path d="M1 1L6 6L11 1" stroke="currentColor"></path></svg><!----></button></div> <!--[-1--><!--]--></div><!----> <!--[0--><h1 class="relative group"><a id="tensor-parallelism" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#tensor-parallelism"><span><svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg><!----></span></a> <span><!---->Tensor parallelism<!----></span></h1><!--]--><!----> <p><a href="https://huggingface.co/spaces/nanotron/ultrascale-playbook?section=tensor_parallelism" rel="nofollow">Tensor parallelism</a> shards the weight matrices of a model across devices. Each device holds a column-wise (<code>"colwise"</code>) or row-wise (<code>"rowwise"</code>) slice of each layer and computes a partial result. An <code>AllReduce</code> or <code>AllGather</code> at the layer boundary combines the partial results into the full output. Unlike <a href="./context_parallelism">context parallelism</a>, it reduces <em>weight</em> memory per device, which helps when a model doesn’t fit on one device. Refer to <a href="../training/distributed_inference">Distributed inference</a> to compare it with the other strategies.</p> <!--[1--><h2 class="relative group"><a id="load-a-sharded-model" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#load-a-sharded-model"><span><svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg><!----></span></a> <span><!---->Load a sharded model<!----></span></h2><!--]--><!----> <p>Pass a <a href="/docs/diffusers/pr_14972/en/api/parallel#diffusers.TensorParallelConfig">TensorParallelConfig</a> to the <code>parallel_config</code> argument of the model’s <a href="/docs/diffusers/pr_14972/en/api/models/overview#diffusers.ModelMixin.from_pretrained">from_pretrained()</a>. <code>tp_degree</code> is the number of devices to shard across and must divide the model’s number of attention heads. The model’s class must define a <code>_tp_plan</code>, which maps module names to a sharding style. To check whether a model supports tensor parallelism, look for this attribute on its class, such as <a href="/docs/diffusers/pr_14972/en/api/models/flux2_transformer#diffusers.Flux2Transformer2DModel">Flux2Transformer2DModel</a>.</p> <p>To add a plan to another model, see <a href="#add-tensor-parallelism-to-a-model">Add tensor parallelism to a model</a>.</p> <p>Loading this way shards the checkpoint as it’s read. Each rank reads only its own slice of each weight and places it directly on its GPU, so no rank ever holds a full-size weight. Memory per rank drops as <code>tp_degree</code> increases.</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5   text-gray-600  " title="code excerpt" type="button"><svg xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg><!----> <div class=" absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0 "><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent;"></div> Copied</div><!----></button><!----></div> <pre class="language-py "><!----><span class="hljs-keyword">import</span> os
<span class="hljs-keyword">import</span> torch
<span class="hljs-keyword">from</span> torch <span class="hljs-keyword">import</span> distributed <span class="hljs-keyword">as</span> dist
<span class="hljs-keyword">from</span> diffusers <span class="hljs-keyword">import</span> DiffusionPipeline, Flux2Transformer2DModel, TensorParallelConfig

<span class="hljs-keyword">def</span> <span class="hljs-title function_">main</span>():
    dist.init_process_group(backend=<span class="hljs-string">&quot;nccl&quot;</span>)
    rank, world_size = dist.get_rank(), dist.get_world_size()
    <span class="hljs-comment"># torchrun sets LOCAL_RANK to this process&#x27;s GPU index on its node</span>
    local_rank = <span class="hljs-built_in">int</span>(os.environ[<span class="hljs-string">&quot;LOCAL_RANK&quot;</span>])
    device = torch.device(<span class="hljs-string">f&quot;cuda:<span class="hljs-subst">{local_rank}</span>&quot;</span>)
    torch.cuda.set_device(device)

    <span class="hljs-comment"># Each rank reads only its own shard of every planned weight, straight onto its own GPU.</span>
    transformer = Flux2Transformer2DModel.from_pretrained(
        <span class="hljs-string">&quot;black-forest-labs/FLUX.2-dev&quot;</span>,
        subfolder=<span class="hljs-string">&quot;transformer&quot;</span>,
        dtype=torch.bfloat16,
        parallel_config=TensorParallelConfig(tp_degree=world_size),
    )

    pipeline = DiffusionPipeline.from_pretrained(
        <span class="hljs-string">&quot;black-forest-labs/FLUX.2-dev&quot;</span>, transformer=transformer, dtype=torch.bfloat16
    )
    <span class="hljs-comment"># The transformer is already on its device; move the remaining components individually. Do not call</span>
    <span class="hljs-comment"># `pipeline.to(device)` — that would move every rank&#x27;s shards onto the same device.</span>
    pipeline.text_encoder.to(device)
    pipeline.vae.to(device)

    generator = torch.Generator().manual_seed(<span class="hljs-number">42</span>)
    image = pipeline(prompt=<span class="hljs-string">&quot;a cat holding a sign that says hello&quot;</span>, generator=generator).images[<span class="hljs-number">0</span>]
    <span class="hljs-keyword">if</span> rank == <span class="hljs-number">0</span>:
        image.save(<span class="hljs-string">&quot;output.png&quot;</span>)
    dist.destroy_process_group()

<span class="hljs-keyword">if</span> __name__ == <span class="hljs-string">&quot;__main__&quot;</span>:
    main()<!----></pre></div><!----> <p>Save the script as <code>tensor_parallel_flux2.py</code> and launch it with <code>torchrun</code>.</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5   text-gray-600  " title="code excerpt" type="button"><svg xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg><!----> <div class=" absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0 "><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent;"></div> Copied</div><!----></button><!----></div> <pre class="language-shell "><!---->torchrun --nproc-per-node 4 tensor_parallel_flux2.py<!----></pre></div><!----> <p>The script sets <code>tp_degree</code> to <code>world_size</code>, so <code>--nproc-per-node 4</code> shards the transformer across 4 GPUs.</p> <p>Only the transformer is sharded. Each rank holds a full copy of the text encoder, so each GPU needs enough memory for the text encoder plus its share of the transformer.</p> <!--[1--><h2 class="relative group"><a id="loading-performance" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#loading-performance"><span><svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg><!----></span></a> <span><!---->Loading performance<!----></span></h2><!--]--><!----> <p>Loading with <code>parallel_config</code> is faster and uses less CPU memory per rank than loading the full model and then calling <code>enable_parallelism()</code>. The table below shows results for <a href="https://huggingface.co/black-forest-labs/FLUX.2-dev" rel="nofollow">black-forest-labs/FLUX.2-dev</a> (transformer only, 32B params, bf16) on 4x A10G (23GB).</p> <table><thead><tr><th>tp_degree</th><th>method</th><th>load time</th><th>peak CPU/rank</th></tr></thead><tbody><tr><td>4</td><td><code>from_pretrained(parallel_config=...)</code></td><td>12.5s</td><td>6.8GB</td></tr><tr><td>4</td><td><code>from_pretrained</code> + <code>enable_parallelism</code></td><td>30.4s</td><td>64.1GB</td></tr></tbody></table> <!--[1--><h2 class="relative group"><a id="limitations" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#limitations"><span><svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg><!----></span></a> <span><!---->Limitations<!----></span></h2><!--]--><!----> <p>Tensor parallelism doesn’t work with some loading options and features, and you can’t save a sharded model.</p> <ul><li>Loading with a tensor-parallel <code>parallel_config</code> doesn’t work with <code>device_map</code>, <code>quantization_config</code>, <code>low_cpu_mem_usage=False</code>, <code>use_flashpack=True</code>, or non-safetensors weights. Each one raises an error instead of quietly loading the full checkpoint.</li> <li>Combining tensor parallelism with quantization, offloading, or LoRA adapters also raises an error, no matter how the model is sharded.</li> <li>To shard a model that’s already in memory, call <code>enable_parallelism()</code> with the same config. It loads the full model first and then shards it, so every rank needs memory for the full checkpoint. Loading with <code>parallel_config</code> avoids that cost.</li> <li>Saving a tensor-parallel model raises an error in <a href="/docs/diffusers/pr_14972/en/api/models/overview#diffusers.ModelMixin.save_pretrained">save_pretrained()</a>. Save the model before sharding it.</li></ul> <!--[1--><h2 class="relative group"><a id="add-tensor-parallelism-to-a-model" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#add-tensor-parallelism-to-a-model"><span><svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg><!----></span></a> <span><!---->Add tensor parallelism to a model<!----></span></h2><!--]--><!----> <p>This section is for contributors who want to add tensor parallelism support to a Diffusers model.</p> <p>Tensor parallelism works only on models that define a <code>_tp_plan</code>, a class attribute that maps module names to a sharding style. Writing one mostly means pairing each projection that <em>expands</em> the hidden dimension with the projection that <em>contracts</em> it back.</p> <p>Each key can contain at most one <code>*</code>, and the part before the <code>*</code> must resolve to an <a href="https://pytorch.org/docs/stable/generated/torch.nn.ModuleList.html" rel="nofollow"><code>nn.ModuleList</code></a>, so one entry covers every block. A key without a <code>*</code> applies to the model itself. Paths are relative to the model.</p> <!--[2--><h3 class="relative group"><a id="colwise-and-rowwise" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#colwise-and-rowwise"><span><svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg><!----></span></a> <span><!---->Colwise and rowwise<!----></span></h3><!--]--><!----> <table><thead><tr><th>Style</th><th>Shards</th><th>Each rank</th><th>Use for</th></tr></thead><tbody><tr><td><code>"colwise"</code></td><td>output features (<code>weight</code> dim 0)</td><td>computes a slice of the output</td><td><code>to_q</code>, <code>to_k</code>, <code>to_v</code>, FFN in-projection</td></tr><tr><td><code>"rowwise"</code></td><td>input features (<code>weight</code> dim 1)</td><td>computes a partial sum</td><td><code>to_out.0</code>, FFN out-projection</td></tr><tr><td><code>"rowwise_split_input"</code></td><td>input features (<code>weight</code> dim 1)</td><td>computes a partial sum from its slice of the replicated input</td><td>a standalone projection whose input is replicated, like an AdaLN modulation projection that reads the timestep embedding</td></tr></tbody></table> <p>Always pair <code>"colwise"</code> and <code>"rowwise"</code> in that order. A <code>"colwise"</code> projection leaves its output sharded, the following <code>"rowwise"</code> projection consumes that shard directly, and a single <code>AllReduce</code> at the block boundary reconstructs the result. Sharding the pair any other way forces a gather in the middle and communicates far more.</p> <p>In attention, the column-wise Q, K and V projections give each rank a subset of heads, so <code>tp_degree</code> must divide the head count. Encoder-stream duplicates (<code>add_q_proj</code>, <code>to_add_out</code>, <code>ff_context</code>) follow the same pattern as their image-stream counterparts.</p> <!--[2--><h3 class="relative group"><a id="fused-projections" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#fused-projections"><span><svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg><!----></span></a> <span><!---->Fused projections<!----></span></h3><!--]--><!----> <p>When one <code>Linear</code> packs several logical tensors along the dimension being sharded, plain <code>"colwise"</code>/<code>"rowwise"</code> slices straight across the concatenation and misaligns the pieces. Use <code>PackedColwiseParallel</code>/<code>PackedRowwiseParallel</code> instead, which shard each packed block independently.</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5   text-gray-600  " title="code excerpt" type="button"><svg xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg><!----> <div class=" absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0 "><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent;"></div> Copied</div><!----></button><!----></div> <pre class="language-py "><!----><span class="hljs-comment"># in src/diffusers/models/transformers/your_model.py</span>
<span class="hljs-keyword">from</span> ...hooks.tensor_parallel <span class="hljs-keyword">import</span> PackedColwiseParallel, PackedRowwiseParallel<!----></pre></div><!----> <p>Pass <code>blocks</code>, a list of proportional integers whose sum divides the packed dimension. For example, use <code>[1, 1]</code> for a SwiGLU gate and up projection of equal halves, or <code>[1, 1, 1, 3, 3]</code> for a fused Q, K, V, gate and up projection with <code>mlp_ratio=3</code>.</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5   text-gray-600  " title="code excerpt" type="button"><svg xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg><!----> <div class=" absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0 "><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent;"></div> Copied</div><!----></button><!----></div> <pre class="language-py "><!----><span class="hljs-string">&quot;transformer_blocks.*.ff.linear_in&quot;</span>: PackedColwiseParallel([<span class="hljs-number">1</span>, <span class="hljs-number">1</span>]),<!----></pre></div><!----> <p>When the block sizes are only known from the config, omit <code>blocks</code> and store the absolute sizes on the <code>Linear</code> during <code>__init__</code> instead, as <code>_tp_packed_col_blocks</code> or <code>_tp_packed_row_blocks</code>.</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5   text-gray-600  " title="code excerpt" type="button"><svg xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg><!----> <div class=" absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0 "><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent;"></div> Copied</div><!----></button><!----></div> <pre class="language-py "><!----><span class="hljs-comment"># in the attention module&#x27;s __init__</span>
<span class="hljs-variable language_">self</span>.to_out._tp_packed_row_blocks = [<span class="hljs-variable language_">self</span>.inner_dim, <span class="hljs-variable language_">self</span>.mlp_hidden_dim]<!----></pre></div><!----> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5   text-gray-600  " title="code excerpt" type="button"><svg xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg><!----> <div class=" absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0 "><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent;"></div> Copied</div><!----></button><!----></div> <pre class="language-py "><!----><span class="hljs-comment"># in the model&#x27;s _tp_plan</span>
<span class="hljs-string">&quot;single_transformer_blocks.*.attn.to_out&quot;</span>: PackedRowwiseParallel(),<!----></pre></div><!----> <p>The plan in <a href="https://github.com/huggingface/diffusers/blob/main/src/diffusers/models/transformers/transformer_flux2.py" rel="nofollow"><code>transformer_flux2.py</code></a> uses both forms. The double-stream SwiGLU pair passes <code>blocks</code> directly. The fused single-stream QKV and MLP projection stores its sizes on the <code>Linear</code> in <code>Flux2ParallelSelfAttention.__init__</code>, because they depend on <code>mlp_ratio</code> and <code>mlp_mult_factor</code>.</p> <!--[2--><h3 class="relative group"><a id="what-to-leave-out" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#what-to-leave-out"><span><svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg><!----></span></a> <span><!---->What to leave out<!----></span></h3><!--]--><!----> <p>Modules left out of the plan are replicated on every rank. Leave out normalization layers, patch and text embeddings, and the final <code>norm_out</code> and <code>proj_out</code>. They’re small, so sharding them saves little memory and adds communication. AdaLN modulation (<code>img_mod</code> and <code>txt_mod</code>) is usually small enough to leave out too. When it’s large, like MiniMax-H3’s <code>adaln_proj.linear</code>, shard it with <code>"rowwise_split_input"</code>.</p> <!--[2--><h3 class="relative group"><a id="constraints-and-verification" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#constraints-and-verification"><span><svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg><!----></span></a> <span><!---->Constraints and verification<!----></span></h3><!--]--><!----> <ul><li><code>tp_degree</code> must divide <code>config.num_attention_heads</code>.</li> <li>Every packed block must <em>individually</em> be divisible by <code>tp_degree</code>, not just their sum.</li></ul> <p><a href="/docs/diffusers/pr_14972/en/api/models/overview#diffusers.ModelMixin.from_pretrained">from_pretrained()</a> and <code>enable_parallelism()</code> check both before loading or sharding any weight.</p> <p>Validate a new plan numerically rather than by eye: generate with a fixed seed on a single device, then again under tensor parallelism, and compare the outputs. A misplaced <code>"colwise"</code>/<code>"rowwise"</code> usually still runs and produces a plausible but wrong image.</p> <blockquote class="tip"><p>Start from an existing plan for a similar architecture.</p> <ul><li><a href="https://github.com/huggingface/diffusers/blob/main/src/diffusers/models/transformers/transformer_qwenimage.py" rel="nofollow"><code>transformer_qwenimage.py</code></a> has no fused projections, so every entry is plain <code>"colwise"</code> or <code>"rowwise"</code>.</li> <li><a href="https://github.com/huggingface/diffusers/blob/main/src/diffusers/models/transformers/transformer_flux.py" rel="nofollow"><code>transformer_flux.py</code></a> adds one packed row-wise projection.</li> <li><a href="https://github.com/huggingface/diffusers/blob/main/src/diffusers/models/transformers/transformer_flux2.py" rel="nofollow"><code>transformer_flux2.py</code></a> uses both packed styles.</li></ul></blockquote> <a class="!text-gray-400 !no-underline text-sm flex items-center not-prose mt-4" href="https://github.com/huggingface/diffusers/blob/main/docs/source/en/optimization/tensor_parallelism.md" target="_blank"><svg class="mr-1" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M31,16l-7,7l-1.41-1.41L28.17,16l-5.58-5.59L24,9l7,7z"></path><path d="M1,16l7-7l1.41,1.41L3.83,16l5.58,5.59L8,23l-7-7z"></path><path d="M12.419,25.484L17.639,6.552l1.932,0.518L14.351,26.002z"></path></svg><!----> <span><span class="underline">Update</span> on GitHub</span></a><!----> <p></p><!--]--><!----><!--]--><!--]--><!--]--> <!--[-1--><!--]--><!--]-->
			
			<script>
				{
					__sveltekit_1qsdbwf = {
						base: "/docs/diffusers/main/en",
						assets: "/docs/diffusers/main/en"
					};
					const element = document.currentScript.parentElement;
					Promise.all([
						import("/docs/diffusers/main/en/_app/immutable/entry/start.0S0_iEoG.js"),
						import("/docs/diffusers/main/en/_app/immutable/entry/app.CPdOxCbw.js")
					]).then(([kit, app]) => {
						kit.start(app, element, {
							node_ids: [0, 347],
							data: [null,null],
							form: null,
							error: null
						});
					});
				}
			</script>
		
Xet Storage Details

Size:: 38.1 kB
Xet hash:: 283fa59c3194f8c61181749393876d245c83b998888b29fbe3a74abae6502def
Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.