| 1 | /* |
| 2 | * Copyright (c) 2008 Siarhei Siamashka <ssvb@users.sourceforge.net> |
| 3 | * |
| 4 | * This file is part of FFmpeg |
| 5 | * |
| 6 | * FFmpeg is free software; you can redistribute it and/or |
| 7 | * modify it under the terms of the GNU Lesser General Public |
| 8 | * License as published by the Free Software Foundation; either |
| 9 | * version 2.1 of the License, or (at your option) any later version. |
| 10 | * |
| 11 | * FFmpeg is distributed in the hope that it will be useful, |
| 12 | * but WITHOUT ANY WARRANTY; without even the implied warranty of |
| 13 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU |
| 14 | * Lesser General Public License for more details. |
| 15 | * |
| 16 | * You should have received a copy of the GNU Lesser General Public |
| 17 | * License along with FFmpeg; if not, write to the Free Software |
| 18 | * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA |
| 19 | */ |
| 20 | |
| 21 | #include "config.h" |
| 22 | #include "asm.S" |
| 23 | |
| 24 | /** |
| 25 | * Assume that len is a positive number and is multiple of 8 |
| 26 | */ |
| 27 | @ void ff_vector_fmul_vfp(float *dst, const float *src0, const float *src1, int len) |
| 28 | function ff_vector_fmul_vfp, export=1 |
| 29 | vpush {d8-d15} |
| 30 | fmrx r12, fpscr |
| 31 | orr r12, r12, #(3 << 16) /* set vector size to 4 */ |
| 32 | fmxr fpscr, r12 |
| 33 | |
| 34 | vldmia r1!, {s0-s3} |
| 35 | vldmia r2!, {s8-s11} |
| 36 | vldmia r1!, {s4-s7} |
| 37 | vldmia r2!, {s12-s15} |
| 38 | vmul.f32 s8, s0, s8 |
| 39 | 1: |
| 40 | subs r3, r3, #16 |
| 41 | vmul.f32 s12, s4, s12 |
| 42 | itttt ge |
| 43 | vldmiage r1!, {s16-s19} |
| 44 | vldmiage r2!, {s24-s27} |
| 45 | vldmiage r1!, {s20-s23} |
| 46 | vldmiage r2!, {s28-s31} |
| 47 | it ge |
| 48 | vmulge.f32 s24, s16, s24 |
| 49 | vstmia r0!, {s8-s11} |
| 50 | vstmia r0!, {s12-s15} |
| 51 | it ge |
| 52 | vmulge.f32 s28, s20, s28 |
| 53 | itttt gt |
| 54 | vldmiagt r1!, {s0-s3} |
| 55 | vldmiagt r2!, {s8-s11} |
| 56 | vldmiagt r1!, {s4-s7} |
| 57 | vldmiagt r2!, {s12-s15} |
| 58 | ittt ge |
| 59 | vmulge.f32 s8, s0, s8 |
| 60 | vstmiage r0!, {s24-s27} |
| 61 | vstmiage r0!, {s28-s31} |
| 62 | bgt 1b |
| 63 | |
| 64 | bic r12, r12, #(7 << 16) /* set vector size back to 1 */ |
| 65 | fmxr fpscr, r12 |
| 66 | vpop {d8-d15} |
| 67 | bx lr |
| 68 | endfunc |
| 69 | |
| 70 | /** |
| 71 | * ARM VFP implementation of 'vector_fmul_window_c' function |
| 72 | * Assume that len is a positive non-zero number |
| 73 | */ |
| 74 | @ void ff_vector_fmul_window_vfp(float *dst, const float *src0, |
| 75 | @ const float *src1, const float *win, int len) |
| 76 | function ff_vector_fmul_window_vfp, export=1 |
| 77 | DST0 .req a1 |
| 78 | SRC0 .req a2 |
| 79 | SRC1 .req a3 |
| 80 | WIN0 .req a4 |
| 81 | LEN .req v1 |
| 82 | DST1 .req v2 |
| 83 | WIN1 .req v3 |
| 84 | OLDFPSCR .req ip |
| 85 | |
| 86 | push {v1-v3,lr} |
| 87 | ldr LEN, [sp, #4*4+0] |
| 88 | vpush {s16-s31} |
| 89 | fmrx OLDFPSCR, FPSCR |
| 90 | add DST1, DST0, LEN, lsl #3 |
| 91 | add SRC1, SRC1, LEN, lsl #2 |
| 92 | add WIN1, WIN0, LEN, lsl #3 |
| 93 | |
| 94 | tst LEN, #7 |
| 95 | beq 4f @ common case: len is a multiple of 8 |
| 96 | |
| 97 | ldr lr, =0x03000000 @ RunFast mode, scalar mode |
| 98 | fmxr FPSCR, lr |
| 99 | |
| 100 | tst LEN, #1 |
| 101 | beq 1f |
| 102 | vldmdb WIN1!, {s0} |
| 103 | vldmia SRC0!, {s8} |
| 104 | vldmia WIN0!, {s16} |
| 105 | vmul.f s24, s0, s8 |
| 106 | vldmdb SRC1!, {s20} |
| 107 | vmul.f s8, s16, s8 |
| 108 | vmls.f s24, s16, s20 |
| 109 | vmla.f s8, s0, s20 |
| 110 | vstmia DST0!, {s24} |
| 111 | vstmdb DST1!, {s8} |
| 112 | 1: |
| 113 | tst LEN, #2 |
| 114 | beq 2f |
| 115 | vldmdb WIN1!, {s0} |
| 116 | vldmdb WIN1!, {s1} |
| 117 | vldmia SRC0!, {s8-s9} |
| 118 | vldmia WIN0!, {s16-s17} |
| 119 | vmul.f s24, s0, s8 |
| 120 | vmul.f s25, s1, s9 |
| 121 | vldmdb SRC1!, {s20} |
| 122 | vldmdb SRC1!, {s21} |
| 123 | vmul.f s8, s16, s8 |
| 124 | vmul.f s9, s17, s9 |
| 125 | vmls.f s24, s16, s20 |
| 126 | vmls.f s25, s17, s21 |
| 127 | vmla.f s8, s0, s20 |
| 128 | vmla.f s9, s1, s21 |
| 129 | vstmia DST0!, {s24-s25} |
| 130 | vstmdb DST1!, {s8} |
| 131 | vstmdb DST1!, {s9} |
| 132 | 2: |
| 133 | tst LEN, #4 |
| 134 | beq 3f |
| 135 | vldmdb WIN1!, {s0} |
| 136 | vldmdb WIN1!, {s1} |
| 137 | vldmdb WIN1!, {s2} |
| 138 | vldmdb WIN1!, {s3} |
| 139 | vldmia SRC0!, {s8-s11} |
| 140 | vldmia WIN0!, {s16-s19} |
| 141 | vmul.f s24, s0, s8 |
| 142 | vmul.f s25, s1, s9 |
| 143 | vmul.f s26, s2, s10 |
| 144 | vmul.f s27, s3, s11 |
| 145 | vldmdb SRC1!, {s20} |
| 146 | vldmdb SRC1!, {s21} |
| 147 | vldmdb SRC1!, {s22} |
| 148 | vldmdb SRC1!, {s23} |
| 149 | vmul.f s8, s16, s8 |
| 150 | vmul.f s9, s17, s9 |
| 151 | vmul.f s10, s18, s10 |
| 152 | vmul.f s11, s19, s11 |
| 153 | vmls.f s24, s16, s20 |
| 154 | vmls.f s25, s17, s21 |
| 155 | vmls.f s26, s18, s22 |
| 156 | vmls.f s27, s19, s23 |
| 157 | vmla.f s8, s0, s20 |
| 158 | vmla.f s9, s1, s21 |
| 159 | vmla.f s10, s2, s22 |
| 160 | vmla.f s11, s3, s23 |
| 161 | vstmia DST0!, {s24-s27} |
| 162 | vstmdb DST1!, {s8} |
| 163 | vstmdb DST1!, {s9} |
| 164 | vstmdb DST1!, {s10} |
| 165 | vstmdb DST1!, {s11} |
| 166 | 3: |
| 167 | bics LEN, LEN, #7 |
| 168 | beq 7f |
| 169 | 4: |
| 170 | ldr lr, =0x03030000 @ RunFast mode, short vectors of length 4, stride 1 |
| 171 | fmxr FPSCR, lr |
| 172 | |
| 173 | vldmdb WIN1!, {s0} |
| 174 | vldmdb WIN1!, {s1} |
| 175 | vldmdb WIN1!, {s2} |
| 176 | vldmdb WIN1!, {s3} |
| 177 | vldmia SRC0!, {s8-s11} |
| 178 | vldmia WIN0!, {s16-s19} |
| 179 | vmul.f s24, s0, s8 @ vector * vector |
| 180 | vldmdb SRC1!, {s20} |
| 181 | vldmdb SRC1!, {s21} |
| 182 | vldmdb SRC1!, {s22} |
| 183 | vldmdb SRC1!, {s23} |
| 184 | vmul.f s8, s16, s8 @ vector * vector |
| 185 | vmls.f s24, s16, s20 @ vector * vector |
| 186 | vldmdb WIN1!, {s4} |
| 187 | vldmdb WIN1!, {s5} |
| 188 | vldmdb WIN1!, {s6} |
| 189 | vldmdb WIN1!, {s7} |
| 190 | vldmia SRC0!, {s12-s13} |
| 191 | vmla.f s8, s0, s20 @ vector * vector |
| 192 | vldmia SRC0!, {s14-s15} |
| 193 | subs LEN, LEN, #8 |
| 194 | beq 6f |
| 195 | 5: vldmia WIN0!, {s20-s23} |
| 196 | vmul.f s28, s4, s12 @ vector * vector |
| 197 | vstmia DST0!, {s24-s25} |
| 198 | vldmdb SRC1!, {s16} |
| 199 | vldmdb SRC1!, {s17} |
| 200 | vldmdb SRC1!, {s18} |
| 201 | vldmdb SRC1!, {s19} |
| 202 | vmul.f s12, s20, s12 @ vector * vector |
| 203 | vstmia DST0!, {s26-s27} |
| 204 | vstmdb DST1!, {s8} |
| 205 | vstmdb DST1!, {s9} |
| 206 | vstmdb DST1!, {s10} |
| 207 | vstmdb DST1!, {s11} |
| 208 | vmls.f s28, s20, s16 @ vector * vector |
| 209 | vldmdb WIN1!, {s0} |
| 210 | vldmdb WIN1!, {s1} |
| 211 | vldmdb WIN1!, {s2} |
| 212 | vldmdb WIN1!, {s3} |
| 213 | vldmia SRC0!, {s8-s9} |
| 214 | vmla.f s12, s4, s16 @ vector * vector |
| 215 | vldmia SRC0!, {s10-s11} |
| 216 | subs LEN, LEN, #8 |
| 217 | vldmia WIN0!, {s16-s19} |
| 218 | vmul.f s24, s0, s8 @ vector * vector |
| 219 | vstmia DST0!, {s28-s29} |
| 220 | vldmdb SRC1!, {s20} |
| 221 | vldmdb SRC1!, {s21} |
| 222 | vldmdb SRC1!, {s22} |
| 223 | vldmdb SRC1!, {s23} |
| 224 | vmul.f s8, s16, s8 @ vector * vector |
| 225 | vstmia DST0!, {s30-s31} |
| 226 | vstmdb DST1!, {s12} |
| 227 | vstmdb DST1!, {s13} |
| 228 | vstmdb DST1!, {s14} |
| 229 | vstmdb DST1!, {s15} |
| 230 | vmls.f s24, s16, s20 @ vector * vector |
| 231 | vldmdb WIN1!, {s4} |
| 232 | vldmdb WIN1!, {s5} |
| 233 | vldmdb WIN1!, {s6} |
| 234 | vldmdb WIN1!, {s7} |
| 235 | vldmia SRC0!, {s12-s13} |
| 236 | vmla.f s8, s0, s20 @ vector * vector |
| 237 | vldmia SRC0!, {s14-s15} |
| 238 | bne 5b |
| 239 | 6: vldmia WIN0!, {s20-s23} |
| 240 | vmul.f s28, s4, s12 @ vector * vector |
| 241 | vstmia DST0!, {s24-s25} |
| 242 | vldmdb SRC1!, {s16} |
| 243 | vldmdb SRC1!, {s17} |
| 244 | vldmdb SRC1!, {s18} |
| 245 | vldmdb SRC1!, {s19} |
| 246 | vmul.f s12, s20, s12 @ vector * vector |
| 247 | vstmia DST0!, {s26-s27} |
| 248 | vstmdb DST1!, {s8} |
| 249 | vstmdb DST1!, {s9} |
| 250 | vstmdb DST1!, {s10} |
| 251 | vstmdb DST1!, {s11} |
| 252 | vmls.f s28, s20, s16 @ vector * vector |
| 253 | vmla.f s12, s4, s16 @ vector * vector |
| 254 | vstmia DST0!, {s28-s31} |
| 255 | vstmdb DST1!, {s12} |
| 256 | vstmdb DST1!, {s13} |
| 257 | vstmdb DST1!, {s14} |
| 258 | vstmdb DST1!, {s15} |
| 259 | 7: |
| 260 | fmxr FPSCR, OLDFPSCR |
| 261 | vpop {s16-s31} |
| 262 | pop {v1-v3,pc} |
| 263 | |
| 264 | .unreq DST0 |
| 265 | .unreq SRC0 |
| 266 | .unreq SRC1 |
| 267 | .unreq WIN0 |
| 268 | .unreq LEN |
| 269 | .unreq OLDFPSCR |
| 270 | .unreq DST1 |
| 271 | .unreq WIN1 |
| 272 | endfunc |
| 273 | |
| 274 | /** |
| 275 | * ARM VFP optimized implementation of 'vector_fmul_reverse_c' function. |
| 276 | * Assume that len is a positive number and is multiple of 8 |
| 277 | */ |
| 278 | @ void ff_vector_fmul_reverse_vfp(float *dst, const float *src0, |
| 279 | @ const float *src1, int len) |
| 280 | function ff_vector_fmul_reverse_vfp, export=1 |
| 281 | vpush {d8-d15} |
| 282 | add r2, r2, r3, lsl #2 |
| 283 | vldmdb r2!, {s0-s3} |
| 284 | vldmia r1!, {s8-s11} |
| 285 | vldmdb r2!, {s4-s7} |
| 286 | vldmia r1!, {s12-s15} |
| 287 | vmul.f32 s8, s3, s8 |
| 288 | vmul.f32 s9, s2, s9 |
| 289 | vmul.f32 s10, s1, s10 |
| 290 | vmul.f32 s11, s0, s11 |
| 291 | 1: |
| 292 | subs r3, r3, #16 |
| 293 | it ge |
| 294 | vldmdbge r2!, {s16-s19} |
| 295 | vmul.f32 s12, s7, s12 |
| 296 | it ge |
| 297 | vldmiage r1!, {s24-s27} |
| 298 | vmul.f32 s13, s6, s13 |
| 299 | it ge |
| 300 | vldmdbge r2!, {s20-s23} |
| 301 | vmul.f32 s14, s5, s14 |
| 302 | it ge |
| 303 | vldmiage r1!, {s28-s31} |
| 304 | vmul.f32 s15, s4, s15 |
| 305 | it ge |
| 306 | vmulge.f32 s24, s19, s24 |
| 307 | it gt |
| 308 | vldmdbgt r2!, {s0-s3} |
| 309 | it ge |
| 310 | vmulge.f32 s25, s18, s25 |
| 311 | vstmia r0!, {s8-s13} |
| 312 | it ge |
| 313 | vmulge.f32 s26, s17, s26 |
| 314 | it gt |
| 315 | vldmiagt r1!, {s8-s11} |
| 316 | itt ge |
| 317 | vmulge.f32 s27, s16, s27 |
| 318 | vmulge.f32 s28, s23, s28 |
| 319 | it gt |
| 320 | vldmdbgt r2!, {s4-s7} |
| 321 | it ge |
| 322 | vmulge.f32 s29, s22, s29 |
| 323 | vstmia r0!, {s14-s15} |
| 324 | ittt ge |
| 325 | vmulge.f32 s30, s21, s30 |
| 326 | vmulge.f32 s31, s20, s31 |
| 327 | vmulge.f32 s8, s3, s8 |
| 328 | it gt |
| 329 | vldmiagt r1!, {s12-s15} |
| 330 | itttt ge |
| 331 | vmulge.f32 s9, s2, s9 |
| 332 | vmulge.f32 s10, s1, s10 |
| 333 | vstmiage r0!, {s24-s27} |
| 334 | vmulge.f32 s11, s0, s11 |
| 335 | it ge |
| 336 | vstmiage r0!, {s28-s31} |
| 337 | bgt 1b |
| 338 | |
| 339 | vpop {d8-d15} |
| 340 | bx lr |
| 341 | endfunc |
| 342 | |
| 343 | /** |
| 344 | * ARM VFP implementation of 'butterflies_float_c' function |
| 345 | * Assume that len is a positive non-zero number |
| 346 | */ |
| 347 | @ void ff_butterflies_float_vfp(float *restrict v1, float *restrict v2, int len) |
| 348 | function ff_butterflies_float_vfp, export=1 |
| 349 | BASE1 .req a1 |
| 350 | BASE2 .req a2 |
| 351 | LEN .req a3 |
| 352 | OLDFPSCR .req a4 |
| 353 | |
| 354 | vpush {s16-s31} |
| 355 | fmrx OLDFPSCR, FPSCR |
| 356 | |
| 357 | tst LEN, #7 |
| 358 | beq 4f @ common case: len is a multiple of 8 |
| 359 | |
| 360 | ldr ip, =0x03000000 @ RunFast mode, scalar mode |
| 361 | fmxr FPSCR, ip |
| 362 | |
| 363 | tst LEN, #1 |
| 364 | beq 1f |
| 365 | vldmia BASE1!, {s0} |
| 366 | vldmia BASE2!, {s8} |
| 367 | vadd.f s16, s0, s8 |
| 368 | vsub.f s24, s0, s8 |
| 369 | vstr s16, [BASE1, #0-4*1] |
| 370 | vstr s24, [BASE2, #0-4*1] |
| 371 | 1: |
| 372 | tst LEN, #2 |
| 373 | beq 2f |
| 374 | vldmia BASE1!, {s0-s1} |
| 375 | vldmia BASE2!, {s8-s9} |
| 376 | vadd.f s16, s0, s8 |
| 377 | vadd.f s17, s1, s9 |
| 378 | vsub.f s24, s0, s8 |
| 379 | vsub.f s25, s1, s9 |
| 380 | vstr d8, [BASE1, #0-8*1] @ s16,s17 |
| 381 | vstr d12, [BASE2, #0-8*1] @ s24,s25 |
| 382 | 2: |
| 383 | tst LEN, #4 |
| 384 | beq 3f |
| 385 | vldmia BASE1!, {s0-s1} |
| 386 | vldmia BASE2!, {s8-s9} |
| 387 | vldmia BASE1!, {s2-s3} |
| 388 | vldmia BASE2!, {s10-s11} |
| 389 | vadd.f s16, s0, s8 |
| 390 | vadd.f s17, s1, s9 |
| 391 | vsub.f s24, s0, s8 |
| 392 | vsub.f s25, s1, s9 |
| 393 | vadd.f s18, s2, s10 |
| 394 | vadd.f s19, s3, s11 |
| 395 | vsub.f s26, s2, s10 |
| 396 | vsub.f s27, s3, s11 |
| 397 | vstr d8, [BASE1, #0-16*1] @ s16,s17 |
| 398 | vstr d12, [BASE2, #0-16*1] @ s24,s25 |
| 399 | vstr d9, [BASE1, #8-16*1] @ s18,s19 |
| 400 | vstr d13, [BASE2, #8-16*1] @ s26,s27 |
| 401 | 3: |
| 402 | bics LEN, LEN, #7 |
| 403 | beq 7f |
| 404 | 4: |
| 405 | ldr ip, =0x03030000 @ RunFast mode, short vectors of length 4, stride 1 |
| 406 | fmxr FPSCR, ip |
| 407 | |
| 408 | vldmia BASE1!, {s0-s1} |
| 409 | vldmia BASE2!, {s8-s9} |
| 410 | vldmia BASE1!, {s2-s3} |
| 411 | vldmia BASE2!, {s10-s11} |
| 412 | vadd.f s16, s0, s8 |
| 413 | vldmia BASE1!, {s4-s5} |
| 414 | vldmia BASE2!, {s12-s13} |
| 415 | vldmia BASE1!, {s6-s7} |
| 416 | vldmia BASE2!, {s14-s15} |
| 417 | vsub.f s24, s0, s8 |
| 418 | vadd.f s20, s4, s12 |
| 419 | subs LEN, LEN, #8 |
| 420 | beq 6f |
| 421 | 5: vldmia BASE1!, {s0-s3} |
| 422 | vldmia BASE2!, {s8-s11} |
| 423 | vsub.f s28, s4, s12 |
| 424 | vstr d8, [BASE1, #0-16*3] @ s16,s17 |
| 425 | vstr d9, [BASE1, #8-16*3] @ s18,s19 |
| 426 | vstr d12, [BASE2, #0-16*3] @ s24,s25 |
| 427 | vstr d13, [BASE2, #8-16*3] @ s26,s27 |
| 428 | vadd.f s16, s0, s8 |
| 429 | vldmia BASE1!, {s4-s7} |
| 430 | vldmia BASE2!, {s12-s15} |
| 431 | vsub.f s24, s0, s8 |
| 432 | vstr d10, [BASE1, #0-16*3] @ s20,s21 |
| 433 | vstr d11, [BASE1, #8-16*3] @ s22,s23 |
| 434 | vstr d14, [BASE2, #0-16*3] @ s28,s29 |
| 435 | vstr d15, [BASE2, #8-16*3] @ s30,s31 |
| 436 | vadd.f s20, s4, s12 |
| 437 | subs LEN, LEN, #8 |
| 438 | bne 5b |
| 439 | 6: vsub.f s28, s4, s12 |
| 440 | vstr d8, [BASE1, #0-16*2] @ s16,s17 |
| 441 | vstr d9, [BASE1, #8-16*2] @ s18,s19 |
| 442 | vstr d12, [BASE2, #0-16*2] @ s24,s25 |
| 443 | vstr d13, [BASE2, #8-16*2] @ s26,s27 |
| 444 | vstr d10, [BASE1, #0-16*1] @ s20,s21 |
| 445 | vstr d11, [BASE1, #8-16*1] @ s22,s23 |
| 446 | vstr d14, [BASE2, #0-16*1] @ s28,s29 |
| 447 | vstr d15, [BASE2, #8-16*1] @ s30,s31 |
| 448 | 7: |
| 449 | fmxr FPSCR, OLDFPSCR |
| 450 | vpop {s16-s31} |
| 451 | bx lr |
| 452 | |
| 453 | .unreq BASE1 |
| 454 | .unreq BASE2 |
| 455 | .unreq LEN |
| 456 | .unreq OLDFPSCR |
| 457 | endfunc |