Download model/protenix/layer_norm/kernel/layer_norm_cuda.cpp from OneScience-Group/ESM: direct link, hf CLI and curl.
- Browser
- Download file 8.12 kB
-
https://huggingface.co/OneScience-Group/ESM/resolve/main/model/protenix/layer_norm/kernel/layer_norm_cuda.cpp
- Command line
-
hf download hf://OneScience-Group/ESM/model/protenix/layer_norm/kernel/layer_norm_cuda.cpp
-
curl -L -o layer_norm_cuda.cpp https://huggingface.co/OneScience-Group/ESM/resolve/main/model/protenix/layer_norm/kernel/layer_norm_cuda.cpp
8.12 kB
| // Copyright 2021- HPC-AI Technology Inc. | |
| // | |
| // Licensed under the Apache License, Version 2.0 (the "License"); | |
| // you may not use this file except in compliance with the License. | |
| // You may obtain a copy of the License at | |
| // | |
| // http://www.apache.org/licenses/LICENSE-2.0 | |
| // | |
| // Unless required by applicable law or agreed to in writing, software | |
| // distributed under the License is distributed on an "AS IS" BASIS, | |
| // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | |
| // See the License for the specific language governing permissions and | |
| void compute_n1_n2(at::Tensor input, at::IntArrayRef normalized_shape, int& n1, int& n2) { | |
| int idiff = input.ndimension() - normalized_shape.size(); | |
| n2 = 1; | |
| for (int i = 0; i < (int)normalized_shape.size(); ++i) { | |
| assert(input.sizes()[i + idiff] == normalized_shape[i]); | |
| n2 *= normalized_shape[i]; | |
| } | |
| n1 = 1; | |
| for (int i = 0; i < idiff; ++i) { | |
| n1 *= input.sizes()[i]; | |
| } | |
| } | |
| void check_args(at::Tensor input, at::IntArrayRef normalized_shape, int& n1, int& n2) { | |
| int64_t normalized_ndim = normalized_shape.size(); | |
| if (normalized_ndim < 1) { | |
| std::stringstream ss; | |
| ss << "Expected normalized_shape to be at least 1-dimensional, i.e., " | |
| << "containing at least one element, but got normalized_shape=" << normalized_shape; | |
| throw std::runtime_error(ss.str()); | |
| } | |
| auto input_shape = input.sizes(); | |
| auto input_ndim = input.dim(); | |
| if (input_ndim < normalized_ndim || | |
| !input_shape.slice(input_ndim - normalized_ndim).equals(normalized_shape)) { | |
| std::stringstream ss; | |
| ss << "Given normalized_shape=" << normalized_shape << ", expected input with shape [*"; | |
| for (auto size : normalized_shape) { | |
| ss << ", " << size; | |
| } | |
| ss << "], but got input of size" << input_shape; | |
| throw std::runtime_error(ss.str()); | |
| } | |
| compute_n1_n2(input, normalized_shape, n1, n2); | |
| } | |
| void cuda_layer_norm(at::Tensor* output, at::Tensor* mean, at::Tensor* invvar, at::Tensor* input, | |
| int n1, int n2, at::IntArrayRef normalized_shape, at::Tensor* gamma, | |
| at::Tensor* beta, double epsilon); | |
| std::vector<at::Tensor> layer_norm_affine(at::Tensor input, at::IntArrayRef normalized_shape, | |
| at::Tensor *gamma, at::Tensor *beta, double epsilon) { | |
| CHECK_INPUT(input); | |
| // CHECK_INPUT((*gamma)); | |
| // CHECK_INPUT((*beta)); | |
| int n1, n2; | |
| check_args(input, normalized_shape, n1, n2); | |
| const at::cuda::OptionalCUDAGuard device_guard(device_of(input)); | |
| at::Tensor output = at::empty_like(input, input.options().dtype(input.scalar_type())); | |
| at::Tensor mean = at::empty({n1}, input.options().dtype(at::ScalarType::Float)); | |
| at::Tensor invvar = at::empty_like(mean); | |
| cuda_layer_norm(&output, &mean, &invvar, &input, n1, n2, normalized_shape, gamma, beta, epsilon); | |
| return {output, mean, invvar}; | |
| } | |
| void cuda_layer_norm_gradient(at::Tensor* dout, at::Tensor* mean, at::Tensor* invvar, | |
| at::Tensor* input, int n1, int n2, at::IntArrayRef normalized_shape, | |
| at::Tensor* gamma, at::Tensor* beta, double epsilon, | |
| at::Tensor* grad_input, at::Tensor* grad_gamma, | |
| at::Tensor* grad_beta); | |
| std::vector<at::Tensor> layer_norm_gradient_affine(at::Tensor dout, at::Tensor mean, | |
| at::Tensor invvar, at::Tensor input, | |
| at::IntArrayRef normalized_shape, | |
| at::Tensor* gamma, at::Tensor* beta, | |
| double epsilon) { | |
| CHECK_INPUT(dout); | |
| CHECK_INPUT(mean); | |
| CHECK_INPUT(invvar); | |
| CHECK_INPUT(input); | |
| int n1, n2; | |
| check_args(input, normalized_shape, n1, n2); | |
| const at::cuda::OptionalCUDAGuard device_guard(device_of(input)); | |
| at::Tensor grad_input = at::empty_like(input); | |
| at::Tensor grad_gamma; | |
| at::Tensor grad_beta; | |
| if (gamma != NULL) | |
| grad_gamma = at::empty_like(*gamma); | |
| if (beta != NULL) | |
| grad_beta = at::empty_like(*beta); | |
| if (gamma != NULL) { | |
| if(beta != NULL) { | |
| cuda_layer_norm_gradient(&dout, &mean, &invvar, &input, n1, n2, normalized_shape, gamma, beta, | |
| epsilon, &grad_input, &grad_gamma, &grad_beta); | |
| } else { | |
| cuda_layer_norm_gradient(&dout, &mean, &invvar, &input, n1, n2, normalized_shape, gamma, beta, | |
| epsilon, &grad_input, &grad_gamma, NULL); | |
| } | |
| } else { | |
| if(beta != NULL) { | |
| cuda_layer_norm_gradient(&dout, &mean, &invvar, &input, n1, n2, normalized_shape, gamma, beta, | |
| epsilon, &grad_input, NULL, &grad_beta); | |
| } else { | |
| cuda_layer_norm_gradient(&dout, &mean, &invvar, &input, n1, n2, normalized_shape, gamma, beta, | |
| epsilon, &grad_input, NULL, NULL); | |
| } | |
| } | |
| return {grad_input, grad_gamma, grad_beta}; | |
| } | |
| PYBIND11_MODULE(TORCH_EXTENSION_NAME, m) { | |
| m.def("forward_none_affine", [](at::Tensor input, at::IntArrayRef normalized_shape, double epsilon) { | |
| return layer_norm_affine(input, normalized_shape, NULL, NULL, epsilon); | |
| }, "LayerNorm forward (CUDA)"); | |
| m.def("forward_with_bias_affine", [](at::Tensor input, at::IntArrayRef normalized_shape, at::Tensor *beta, double epsilon) { | |
| return layer_norm_affine(input, normalized_shape, NULL, beta, epsilon); | |
| }, "LayerNorm forward (CUDA)"); | |
| m.def("forward_with_weight_affine", [](at::Tensor input, at::IntArrayRef normalized_shape, at::Tensor *gamma, double epsilon) { | |
| return layer_norm_affine(input, normalized_shape, gamma, NULL, epsilon); | |
| }, "LayerNorm forward (CUDA)"); | |
| m.def("forward_with_both_affine", [](at::Tensor input, at::IntArrayRef normalized_shape, at::Tensor *gamma, at::Tensor *beta, double epsilon) { | |
| return layer_norm_affine(input, normalized_shape, gamma, beta, epsilon); | |
| }, "LayerNorm forward (CUDA)"); | |
| m.def("backward_none_affine", [](at::Tensor dout, at::Tensor mean, at::Tensor invvar, at::Tensor input, | |
| at::IntArrayRef normalized_shape, double epsilon) { | |
| return layer_norm_gradient_affine(dout, mean, invvar, input, normalized_shape, NULL, NULL, epsilon); | |
| }, "LayerNorm backward (CUDA)"); | |
| m.def("backward_with_bias_affine", [](at::Tensor dout, at::Tensor mean, at::Tensor invvar, at::Tensor input, | |
| at::IntArrayRef normalized_shape, at::Tensor *beta, double epsilon) { | |
| return layer_norm_gradient_affine(dout, mean, invvar, input, normalized_shape, NULL, beta, epsilon); | |
| }, "LayerNorm backward (CUDA)"); | |
| m.def("backward_with_weight_affine", [](at::Tensor dout, at::Tensor mean, at::Tensor invvar, at::Tensor input, | |
| at::IntArrayRef normalized_shape, at::Tensor *gamma, double epsilon) { | |
| return layer_norm_gradient_affine(dout, mean, invvar, input, normalized_shape, gamma, NULL, epsilon); | |
| }, "LayerNorm backward (CUDA)"); | |
| m.def("backward_with_both_affine", [](at::Tensor dout, at::Tensor mean, at::Tensor invvar, at::Tensor input, | |
| at::IntArrayRef normalized_shape, at::Tensor *gamma, at::Tensor *beta, double epsilon) { | |
| return layer_norm_gradient_affine(dout, mean, invvar, input, normalized_shape, gamma, beta, epsilon); | |
| }, "LayerNorm backward (CUDA)"); | |
| } | |