Download ada_ln_zero.py from InstaDeepAI/CellScribe: direct link, hf CLI and curl.
- Browser
- Download file 7.93 kB
-
https://huggingface.co/InstaDeepAI/CellScribe/resolve/main/ada_ln_zero.py
- Command line
-
hf download hf://InstaDeepAI/CellScribe/ada_ln_zero.py
-
curl -L -o ada_ln_zero.py https://huggingface.co/InstaDeepAI/CellScribe/resolve/main/ada_ln_zero.py
7.93 kB
| # Copyright 2026 InstaDeep Ltd and CentraleSupélec | |
| # | |
| # Licensed under the Apache License, Version 2.0 (the "License"); | |
| # you may not use this file except in compliance with the License. | |
| # You may obtain a copy of the License at | |
| # | |
| # http://www.apache.org/licenses/LICENSE-2.0 | |
| # | |
| # Unless required by applicable law or agreed to in writing, software | |
| # distributed under the License is distributed on an "AS IS" BASIS, | |
| # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | |
| # See the License for the specific language governing permissions and | |
| # limitations under the License. | |
| import types | |
| from typing import Any, Optional, Type, TypeVar | |
| import torch | |
| import torch.nn as nn | |
| from transformers import PreTrainedModel | |
| class AdaLNZero(nn.Module): | |
| """Adaptive layer norm: ``output = norm(x) * (1 + scale) + shift``. | |
| ``(scale, shift) = ada_proj(conditioning_signal)``, computed in fp32. | |
| """ | |
| def __init__(self, original_norm_layer: nn.Module, conditioning_dim: int): | |
| super().__init__() | |
| self.norm = original_norm_layer | |
| hidden_size = original_norm_layer.weight.shape[0] | |
| self.ada_proj = nn.Linear(conditioning_dim, 2 * hidden_size) | |
| nn.init.constant_(self.ada_proj.weight, 0) | |
| nn.init.constant_(self.ada_proj.bias, 0) | |
| # Set by the decoder layer for the duration of its forward pass. | |
| self.conditioning_signal: Optional[torch.Tensor] = None | |
| def forward( | |
| self, x: torch.Tensor, conditioning_signal: Optional[torch.Tensor] = None | |
| ) -> torch.Tensor: | |
| if conditioning_signal is None: | |
| conditioning_signal = self.conditioning_signal | |
| if conditioning_signal is None: | |
| return self.norm(x) | |
| input_dtype = x.dtype | |
| autocast_device = "cuda" if x.device.type == "cuda" else "cpu" | |
| # The modulation is computed in fp32 for stability under bf16/fp16 | |
| # autocast; the AdaLN parameters are kept in fp32. | |
| with torch.amp.autocast(autocast_device, enabled=False): | |
| normed = self.norm(x.float()) | |
| ada_out = self.ada_proj(conditioning_signal.float()) | |
| if ada_out.dim() == 2: | |
| # [B, 2H] -> [B, 1, 2H], broadcast over the sequence. | |
| ada_out = ada_out.unsqueeze(1) | |
| scale, shift = torch.chunk(ada_out, 2, dim=-1) | |
| return (normed * (1 + scale) + shift).to(input_dtype) | |
| def _conditioned_forward( | |
| self, hidden_states, *args, conditioning_signal=None, **kwargs | |
| ): | |
| """The decoder layer's own forward pass, with both norms conditioned.""" | |
| norms = (self.input_layernorm, self.post_attention_layernorm) | |
| for norm in norms: | |
| norm.conditioning_signal = conditioning_signal | |
| try: | |
| return type(self).forward(self, hidden_states, *args, **kwargs) | |
| finally: | |
| for norm in norms: | |
| norm.conditioning_signal = None | |
| def attach_ada_ln_zero(llm: PreTrainedModel, conditioning_dim: int) -> PreTrainedModel: | |
| """Model surgery: replace both norms of every decoder layer with AdaLN-Zero.""" | |
| for layer in llm.model.layers: | |
| for norm_name in ("input_layernorm", "post_attention_layernorm"): | |
| setattr( | |
| layer, | |
| norm_name, | |
| AdaLNZero(getattr(layer, norm_name), conditioning_dim), | |
| ) | |
| layer.forward = types.MethodType(_conditioned_forward, layer) | |
| return llm | |
| T = TypeVar("T", bound=PreTrainedModel) | |
| def create_conditional_model_class(base_model_class: Type[T]) -> Type[T]: | |
| """Subclass whose ``forward`` officially accepts ``conditioning_signal``, so | |
| that ``.generate()`` forwards it to every decoder layer.""" | |
| class ConditionalModel(base_model_class): # type: ignore | |
| # Explicit signature: ``generate`` inspects it (e.g. to derive | |
| # ``position_ids`` from the attention mask under left padding). | |
| def forward( | |
| self, | |
| input_ids: Optional[torch.LongTensor] = None, | |
| attention_mask: Optional[torch.Tensor] = None, | |
| position_ids: Optional[torch.LongTensor] = None, | |
| past_key_values: Optional[list[torch.FloatTensor]] = None, | |
| inputs_embeds: Optional[torch.FloatTensor] = None, | |
| labels: Optional[torch.LongTensor] = None, | |
| use_cache: Optional[bool] = None, | |
| output_attentions: Optional[bool] = None, | |
| output_hidden_states: Optional[bool] = None, | |
| return_dict: Optional[bool] = None, | |
| conditioning_signal: Optional[torch.Tensor] = None, | |
| conditioning_default: Optional[torch.Tensor] = None, | |
| **kwargs, | |
| ): | |
| # A per-position plan ([B, T_plan, D], in-context learning) covers the | |
| # prompt, but under a KV cache each decoding step only forwards the | |
| # newly generated token(s): slice the plan to the positions being | |
| # processed so the conditioning stays aligned with the hidden states. | |
| # A [B, D] signal is broadcast over all positions as is. | |
| if conditioning_signal is not None and conditioning_signal.dim() == 3: | |
| conditioning_signal = self._slice_conditioning_plan( | |
| plan=conditioning_signal, | |
| conditioning_default=conditioning_default, | |
| cache_position=kwargs.get("cache_position"), | |
| position_ids=position_ids, | |
| past_key_values=past_key_values, | |
| ) | |
| return super().forward( | |
| input_ids=input_ids, | |
| attention_mask=attention_mask, | |
| position_ids=position_ids, | |
| past_key_values=past_key_values, | |
| inputs_embeds=inputs_embeds, | |
| labels=labels, | |
| use_cache=use_cache, | |
| output_attentions=output_attentions, | |
| output_hidden_states=output_hidden_states, | |
| return_dict=return_dict, | |
| conditioning_signal=conditioning_signal, | |
| **kwargs, | |
| ) | |
| def _slice_conditioning_plan( | |
| plan: torch.Tensor, | |
| conditioning_default: Optional[torch.Tensor], | |
| cache_position: Optional[torch.Tensor], | |
| position_ids: Optional[torch.LongTensor], | |
| past_key_values: Optional[Any], | |
| ) -> torch.Tensor: | |
| """Select the conditioning rows for the positions being processed. | |
| Positions beyond the plan length (i.e. generated answer tokens) use | |
| ``conditioning_default`` (the query-cell conditioning) when provided, | |
| otherwise the last plan row. | |
| """ | |
| _batch, t_plan, _dim = plan.shape | |
| # Resolve the absolute positions of the current tokens. | |
| if cache_position is not None: | |
| cur = cache_position.to(plan.device) | |
| elif position_ids is not None: | |
| cur = position_ids[0].to(plan.device) | |
| else: | |
| # Fall back to past length + arange over the current chunk. | |
| past_len = 0 | |
| if past_key_values is not None: | |
| get_len = getattr(past_key_values, "get_seq_length", None) | |
| if callable(get_len): | |
| past_len = get_len() | |
| elif len(past_key_values) > 0: | |
| past_len = past_key_values[0][0].shape[-2] | |
| cur = torch.arange(past_len, past_len + t_plan, device=plan.device) | |
| idx = cur.clamp(max=t_plan - 1) | |
| sel = plan[:, idx, :] # [B, n_cur, D] | |
| overflow = cur >= t_plan | |
| if overflow.any() and conditioning_default is not None: | |
| default = conditioning_default.to(sel.dtype) # [B, D] | |
| sel[:, overflow, :] = default.unsqueeze(1) | |
| return sel | |
| return ConditionalModel | |