util/tensor.cpp
| Line | Branch | Exec | Source |
|---|---|---|---|
| 1 | // SPDX-License-Identifier: BSD-3-Clause | ||
| 2 | /// @file tensor.cpp | ||
| 3 | /// @brief Tensor views and DLPack import and export. | ||
| 4 | #include "ernie/util/tensor.h" | ||
| 5 | |||
| 6 | #include <algorithm> | ||
| 7 | #include <cstring> | ||
| 8 | #include <functional> | ||
| 9 | #include <numeric> | ||
| 10 | #include <string> | ||
| 11 | #include <utility> | ||
| 12 | |||
| 13 | namespace ernie { | ||
| 14 | |||
| 15 | namespace { | ||
| 16 | |||
| 17 | 484 | DLDataType ToDl(DType type) { | |
| 18 | 484 | switch (type) { | |
| 19 | 15 | case DType::UInt8: | |
| 20 | 15 | return {kDLUInt, 8, 1}; | |
| 21 | 2 | case DType::UInt16: | |
| 22 | 2 | return {kDLUInt, 16, 1}; | |
| 23 | 189 | case DType::UInt32: | |
| 24 | 189 | return {kDLUInt, 32, 1}; | |
| 25 | 55 | case DType::Int32: | |
| 26 | 55 | return {kDLInt, 32, 1}; | |
| 27 | 17 | case DType::Int64: | |
| 28 | 17 | return {kDLInt, 64, 1}; | |
| 29 | 2 | case DType::Float16: | |
| 30 | 2 | return {kDLFloat, 16, 1}; | |
| 31 | 200 | case DType::Float32: | |
| 32 | 200 | return {kDLFloat, 32, 1}; | |
| 33 | 4 | case DType::Float64: | |
| 34 | 4 | return {kDLFloat, 64, 1}; | |
| 35 | } | ||
| 36 | ✗ | throw std::invalid_argument("Unknown element type"); | |
| 37 | } | ||
| 38 | |||
| 39 | 19 | DType FromDl(DLDataType type) { | |
| 40 | 19 | if (type.lanes != 1) { | |
| 41 | 1 | throw std::invalid_argument("DLPack tensors with vector lanes are not supported"); | |
| 42 | } | ||
| 43 | 18 | if (type.code == kDLUInt && type.bits == 8) { | |
| 44 | return DType::UInt8; | ||
| 45 | } | ||
| 46 | 17 | if (type.code == kDLUInt && type.bits == 16) { | |
| 47 | return DType::UInt16; | ||
| 48 | } | ||
| 49 | 15 | if (type.code == kDLUInt && type.bits == 32) { | |
| 50 | return DType::UInt32; | ||
| 51 | } | ||
| 52 | 14 | if (type.code == kDLInt && type.bits == 32) { | |
| 53 | return DType::Int32; | ||
| 54 | } | ||
| 55 | 12 | if (type.code == kDLInt && type.bits == 64) { | |
| 56 | return DType::Int64; | ||
| 57 | } | ||
| 58 | 11 | if (type.code == kDLFloat && type.bits == 16) { | |
| 59 | return DType::Float16; | ||
| 60 | } | ||
| 61 | 10 | if (type.code == kDLFloat && type.bits == 32) { | |
| 62 | return DType::Float32; | ||
| 63 | } | ||
| 64 | 3 | if (type.code == kDLFloat && type.bits == 64) { | |
| 65 | return DType::Float64; | ||
| 66 | } | ||
| 67 | 2 | throw std::invalid_argument("Unsupported DLPack data type (code " + std::to_string(type.code) + | |
| 68 | 5 | ", " + std::to_string(type.bits) + " bits)"); | |
| 69 | } | ||
| 70 | |||
| 71 | 20 | Device FromDl(DLDevice device) { | |
| 72 | 20 | switch (device.device_type) { | |
| 73 | 10 | case kDLCPU: | |
| 74 | 10 | case kDLCUDAHost: | |
| 75 | 10 | return {DeviceKind::Cpu, 0}; | |
| 76 | 9 | case kDLCUDA: | |
| 77 | 9 | return {DeviceKind::Cuda, device.device_id}; | |
| 78 | ✗ | case kDLROCM: | |
| 79 | ✗ | return {DeviceKind::Rocm, device.device_id}; | |
| 80 | 1 | default: | |
| 81 | 2 | throw std::invalid_argument("Unsupported DLPack device type " + | |
| 82 | 3 | std::to_string(static_cast<int>(device.device_type))); | |
| 83 | } | ||
| 84 | } | ||
| 85 | |||
| 86 | 248 | std::vector<std::int64_t> RowMajor(const std::vector<std::int64_t>& shape) { | |
| 87 | 248 | std::vector<std::int64_t> strides(shape.size(), 1); | |
| 88 | 369 | for (std::size_t i = shape.size(); i > 1; --i) { | |
| 89 | 121 | strides[i - 2] = strides[i - 1] * shape[i - 1]; | |
| 90 | } | ||
| 91 | 248 | return strides; | |
| 92 | } | ||
| 93 | |||
| 94 | /// Wraps a DLPack tensor's view, calling `release` once when the last | ||
| 95 | /// reference goes. | ||
| 96 | template <typename Managed> | ||
| 97 | 20 | Tensor Adopt(Managed* managed, const DLTensor& dl, void (*release)(Managed*)) { | |
| 98 | 20 | std::shared_ptr<const void> owner(managed, [release](Managed* m) { release(m); }); | |
| 99 | 23 | std::vector<std::int64_t> shape(dl.shape, dl.shape + dl.ndim); | |
| 100 | 20 | std::vector<std::int64_t> strides; | |
| 101 | 20 | if (dl.strides != nullptr) { | |
| 102 | 14 | strides.assign(dl.strides, dl.strides + dl.ndim); | |
| 103 | } | ||
| 104 | 20 | void* data = static_cast<std::byte*>(dl.data) + dl.byte_offset; | |
| 105 | 19 | return Tensor(data, FromDl(dl.dtype), std::move(shape), std::move(strides), FromDl(dl.device), | |
| 106 | 42 | std::move(owner)); | |
| 107 | 23 | } | |
| 108 | |||
| 109 | 20 | void ReleaseVersioned(DLManagedTensorVersioned* managed) { | |
| 110 | 20 | if (managed->deleter != nullptr) { | |
| 111 | 20 | managed->deleter(managed); | |
| 112 | } | ||
| 113 | 20 | } | |
| 114 | |||
| 115 | 1 | void ReleaseLegacy(DLManagedTensor* managed) { | |
| 116 | 1 | if (managed->deleter != nullptr) { | |
| 117 | 1 | managed->deleter(managed); | |
| 118 | } | ||
| 119 | 1 | } | |
| 120 | |||
| 121 | /// What an exported tensor keeps alive, freed by its DLPack deleter. | ||
| 122 | struct Export { | ||
| 123 | DLManagedTensorVersioned managed{}; | ||
| 124 | Tensor tensor; | ||
| 125 | std::vector<std::int64_t> shape; | ||
| 126 | std::vector<std::int64_t> strides; | ||
| 127 | }; | ||
| 128 | |||
| 129 | } // namespace | ||
| 130 | |||
| 131 | 473 | std::size_t ByteSize(DType type) { | |
| 132 | 473 | return ToDl(type).bits / 8u; | |
| 133 | } | ||
| 134 | |||
| 135 | 162 | Tensor::Tensor(void* data, DType type, std::vector<std::int64_t> shape, | |
| 136 | 162 | std::vector<std::int64_t> strides, Device device, std::shared_ptr<const void> owner) | |
| 137 | 162 | : data_(data), type_(type), shape_(std::move(shape)), strides_(std::move(strides)), | |
| 138 | 162 | device_(device), owner_(std::move(owner)) { | |
| 139 | 400 | for (const std::int64_t extent : shape_) { | |
| 140 | 239 | if (extent < 0) { | |
| 141 | 1 | throw std::invalid_argument("Tensor dimensions must not be negative"); | |
| 142 | } | ||
| 143 | } | ||
| 144 | 161 | if (strides_.empty()) { | |
| 145 | 144 | strides_ = RowMajor(shape_); | |
| 146 | 17 | } else if (strides_.size() != shape_.size()) { | |
| 147 | 1 | throw std::invalid_argument("Tensor strides must have one entry per dimension"); | |
| 148 | } | ||
| 149 | 162 | } | |
| 150 | |||
| 151 | 133 | Tensor Tensor::Zeros(DType type, std::vector<std::int64_t> shape) { | |
| 152 | 133 | const std::int64_t elements = | |
| 153 | 133 | std::accumulate(shape.begin(), shape.end(), std::int64_t{1}, std::multiplies<>()); | |
| 154 | 133 | if (elements < 0) { | |
| 155 | 1 | throw std::invalid_argument("Tensor dimensions must not be negative"); | |
| 156 | } | ||
| 157 | 132 | const std::size_t bytes = static_cast<std::size_t>(elements) * ByteSize(type); | |
| 158 | 3998448 | std::shared_ptr<std::byte[]> memory(new std::byte[std::max<std::size_t>(bytes, 1)]()); | |
| 159 | 132 | void* data = memory.get(); | |
| 160 | 264 | return Tensor(data, type, std::move(shape), {}, {}, std::move(memory)); | |
| 161 | 132 | } | |
| 162 | |||
| 163 | 21 | Tensor Tensor::FromDlpack(DLManagedTensorVersioned* managed) { | |
| 164 | 21 | if (managed == nullptr) { | |
| 165 | 1 | throw std::invalid_argument("Null DLPack tensor"); | |
| 166 | } | ||
| 167 | 20 | if (managed->version.major != DLPACK_MAJOR_VERSION) { | |
| 168 | 1 | ReleaseVersioned(managed); | |
| 169 | 1 | throw std::invalid_argument("Unsupported DLPack major version"); | |
| 170 | } | ||
| 171 | 19 | return Adopt(managed, managed->dl_tensor, ReleaseVersioned); | |
| 172 | } | ||
| 173 | |||
| 174 | 2 | Tensor Tensor::FromDlpack(DLManagedTensor* managed) { | |
| 175 | 2 | if (managed == nullptr) { | |
| 176 | 1 | throw std::invalid_argument("Null DLPack tensor"); | |
| 177 | } | ||
| 178 | 1 | return Adopt(managed, managed->dl_tensor, ReleaseLegacy); | |
| 179 | } | ||
| 180 | |||
| 181 | 11 | DLManagedTensorVersioned* Tensor::ToDlpack() const { | |
| 182 | 11 | auto* exported = new Export(); | |
| 183 | 11 | exported->tensor = *this; | |
| 184 | 11 | exported->shape = shape_; | |
| 185 | 11 | exported->strides = strides_; | |
| 186 | 11 | DLManagedTensorVersioned& m = exported->managed; | |
| 187 | 11 | m.version = {DLPACK_MAJOR_VERSION, DLPACK_MINOR_VERSION}; | |
| 188 | 11 | m.manager_ctx = exported; | |
| 189 | 22 | m.deleter = [](DLManagedTensorVersioned* self) { | |
| 190 | 11 | delete static_cast<Export*>(self->manager_ctx); | |
| 191 | 11 | }; | |
| 192 | 11 | DLTensor& dl = m.dl_tensor; | |
| 193 | 11 | dl.data = data_; | |
| 194 | 11 | switch (device_.kind) { | |
| 195 | 8 | case DeviceKind::Cuda: | |
| 196 | 8 | dl.device = DLDevice{kDLCUDA, device_.id}; | |
| 197 | 8 | break; | |
| 198 | ✗ | case DeviceKind::Rocm: | |
| 199 | ✗ | dl.device = DLDevice{kDLROCM, device_.id}; | |
| 200 | ✗ | break; | |
| 201 | 3 | default: | |
| 202 | 3 | dl.device = DLDevice{kDLCPU, 0}; | |
| 203 | 3 | break; | |
| 204 | } | ||
| 205 | 11 | dl.ndim = static_cast<std::int32_t>(shape_.size()); | |
| 206 | 11 | dl.dtype = ToDl(type_); | |
| 207 | 11 | dl.shape = exported->shape.data(); | |
| 208 | 11 | dl.strides = exported->strides.data(); | |
| 209 | 11 | dl.byte_offset = 0; | |
| 210 | 11 | return &m; | |
| 211 | } | ||
| 212 | |||
| 213 | 315 | std::int64_t Tensor::Elements() const { | |
| 214 | 315 | return std::accumulate(shape_.begin(), shape_.end(), std::int64_t{1}, std::multiplies<>()); | |
| 215 | } | ||
| 216 | |||
| 217 | 4 | Tensor Tensor::Row(std::int64_t index) const { | |
| 218 | 4 | if (shape_.empty() || index < 0 || index >= shape_[0]) { | |
| 219 | 3 | throw std::out_of_range("Tensor::Row index out of range"); | |
| 220 | } | ||
| 221 | 1 | void* data = static_cast<std::byte*>(data_) + | |
| 222 | 1 | index * strides_[0] * static_cast<std::int64_t>(ByteSize(type_)); | |
| 223 | 2 | return Tensor(data, type_, {shape_.begin() + 1, shape_.end()}, | |
| 224 | 3 | {strides_.begin() + 1, strides_.end()}, device_, owner_); | |
| 225 | } | ||
| 226 | |||
| 227 | ✗ | Tensor Tensor::Slice(std::int64_t first, std::int64_t count) const { | |
| 228 | ✗ | if (shape_.empty() || first < 0 || count < 0 || first + count > shape_[0]) { | |
| 229 | ✗ | throw std::out_of_range("Tensor::Slice entries out of range"); | |
| 230 | } | ||
| 231 | ✗ | void* data = static_cast<std::byte*>(data_) + | |
| 232 | ✗ | first * strides_[0] * static_cast<std::int64_t>(ByteSize(type_)); | |
| 233 | ✗ | std::vector<std::int64_t> shape = shape_; | |
| 234 | ✗ | shape[0] = count; | |
| 235 | ✗ | return Tensor(data, type_, std::move(shape), strides_, device_, owner_); | |
| 236 | ✗ | } | |
| 237 | |||
| 238 | ✗ | Tensor Tensor::Reshaped(std::vector<std::int64_t> shape) const { | |
| 239 | ✗ | std::int64_t elements = 1; | |
| 240 | ✗ | for (const std::int64_t extent : shape) { | |
| 241 | ✗ | elements *= extent; | |
| 242 | } | ||
| 243 | ✗ | if (!IsContiguous() || elements != Elements()) { | |
| 244 | ✗ | throw std::invalid_argument("Tensor::Reshaped needs a contiguous tensor of as many " | |
| 245 | ✗ | "elements"); | |
| 246 | } | ||
| 247 | ✗ | return Tensor(data_, type_, std::move(shape), {}, device_, owner_); | |
| 248 | } | ||
| 249 | |||
| 250 | 104 | bool Tensor::IsContiguous() const { | |
| 251 | 208 | return strides_ == RowMajor(shape_) || Elements() <= 1; | |
| 252 | } | ||
| 253 | |||
| 254 | 99 | Tensor Tensor::Contiguous() const { | |
| 255 | 99 | if (device_.kind != DeviceKind::Cpu) { | |
| 256 | 1 | throw std::invalid_argument("Tensor::Contiguous needs a CPU tensor"); | |
| 257 | } | ||
| 258 | 98 | if (IsContiguous()) { | |
| 259 | 96 | return *this; | |
| 260 | } | ||
| 261 | 2 | Tensor out = Zeros(type_, shape_); | |
| 262 | 2 | const std::size_t element = ByteSize(type_); | |
| 263 | 2 | std::vector<std::int64_t> index(shape_.size(), 0); | |
| 264 | 2 | auto* destination = static_cast<std::byte*>(out.Data()); | |
| 265 | 2 | const auto* source = static_cast<const std::byte*>(data_); | |
| 266 | 14 | for (std::int64_t n = 0; n < Elements(); ++n) { | |
| 267 | std::int64_t offset = 0; | ||
| 268 | 36 | for (std::size_t d = 0; d < shape_.size(); ++d) { | |
| 269 | 24 | offset += index[d] * strides_[d]; | |
| 270 | } | ||
| 271 | 12 | std::memcpy(destination + static_cast<std::size_t>(n) * element, | |
| 272 | 12 | source + offset * static_cast<std::int64_t>(element), element); | |
| 273 | 20 | for (std::size_t d = shape_.size(); d > 0; --d) { | |
| 274 | 18 | if (++index[d - 1] < shape_[d - 1]) { | |
| 275 | break; | ||
| 276 | } | ||
| 277 | 8 | index[d - 1] = 0; | |
| 278 | } | ||
| 279 | } | ||
| 280 | 2 | return out; | |
| 281 | 2 | } | |
| 282 | |||
| 283 | 50 | std::vector<float> CopyFloats(const Tensor& tensor, std::string_view what) { | |
| 284 | 50 | if (tensor.Empty()) { | |
| 285 | 3 | return {}; | |
| 286 | } | ||
| 287 | 47 | if (!tensor.OnHost()) { | |
| 288 | ✗ | throw std::invalid_argument(std::string(what) + " must be a CPU array"); | |
| 289 | } | ||
| 290 | 47 | const Tensor packed = tensor.Contiguous(); | |
| 291 | 47 | const auto count = static_cast<std::size_t>(packed.Elements()); | |
| 292 | 47 | if (packed.Type() == DType::Float32) { | |
| 293 | 47 | return {packed.As<float>(), packed.As<float>() + count}; | |
| 294 | } | ||
| 295 | ✗ | if (packed.Type() == DType::Float64) { | |
| 296 | ✗ | std::vector<float> values(count); | |
| 297 | ✗ | std::transform(packed.As<double>(), packed.As<double>() + count, values.begin(), | |
| 298 | ✗ | [](double value) { return static_cast<float>(value); }); | |
| 299 | ✗ | return values; | |
| 300 | ✗ | } | |
| 301 | ✗ | throw std::invalid_argument(std::string(what) + " must be float32 or float64"); | |
| 302 | 47 | } | |
| 303 | |||
| 304 | 26 | std::vector<float> CopyFloats(const Tensor& tensor, std::size_t components, std::string_view what) { | |
| 305 | 26 | if (tensor.Empty()) { | |
| 306 | 12 | return {}; | |
| 307 | } | ||
| 308 | 14 | const bool shaped = | |
| 309 | 14 | tensor.Rank() == 2 && tensor.Shape()[1] == static_cast<std::int64_t>(components); | |
| 310 | 14 | if (tensor.Location().kind != DeviceKind::Cpu || !(tensor.Rank() == 1 || shaped) || | |
| 311 | 14 | static_cast<std::size_t>(tensor.Elements()) % components != 0) { | |
| 312 | ✗ | throw std::invalid_argument(std::string(what) + " must be a CPU array of " + | |
| 313 | ✗ | std::to_string(components) + "-vectors"); | |
| 314 | } | ||
| 315 | 14 | return CopyFloats(tensor, what); | |
| 316 | } | ||
| 317 | |||
| 318 | 40 | std::vector<std::int64_t> CopyInts(const Tensor& tensor, std::string_view what) { | |
| 319 | 40 | if (tensor.Empty()) { | |
| 320 | 2 | return {}; | |
| 321 | } | ||
| 322 | 38 | if (!tensor.OnHost()) { | |
| 323 | ✗ | throw std::invalid_argument(std::string(what) + " must be a CPU array"); | |
| 324 | } | ||
| 325 | 38 | const Tensor packed = tensor.Contiguous(); | |
| 326 | 38 | const auto count = static_cast<std::size_t>(packed.Elements()); | |
| 327 | 38 | switch (packed.Type()) { | |
| 328 | 38 | case DType::Int32: | |
| 329 | 38 | return {packed.As<std::int32_t>(), packed.As<std::int32_t>() + count}; | |
| 330 | ✗ | case DType::Int64: | |
| 331 | ✗ | return {packed.As<std::int64_t>(), packed.As<std::int64_t>() + count}; | |
| 332 | ✗ | case DType::UInt32: | |
| 333 | ✗ | return {packed.As<std::uint32_t>(), packed.As<std::uint32_t>() + count}; | |
| 334 | ✗ | default: | |
| 335 | ✗ | throw std::invalid_argument(std::string(what) + " must be int32, int64 or uint32"); | |
| 336 | } | ||
| 337 | 38 | } | |
| 338 | |||
| 339 | } // namespace ernie | ||
| 340 |