core/bitblt.cpp
| Line | Branch | Exec | Source |
|---|---|---|---|
| 1 | // This program is free software; you can redistribute it and/or modify | ||
| 2 | // it under the terms of the GNU General Public License as published by | ||
| 3 | // the Free Software Foundation; either version 2 of the License, or | ||
| 4 | // (at your option) any later version. | ||
| 5 | // | ||
| 6 | // This program is distributed in the hope that it will be useful, | ||
| 7 | // but WITHOUT ANY WARRANTY; without even the implied warranty of | ||
| 8 | // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the | ||
| 9 | // GNU General Public License for more details. | ||
| 10 | // | ||
| 11 | // You should have received a copy of the GNU General Public License | ||
| 12 | // along with this program; if not, write to the Free Software | ||
| 13 | // Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA, or visit | ||
| 14 | // http://www.gnu.org/copyleft/gpl.html . | ||
| 15 | // | ||
| 16 | // Linking Avisynth statically or dynamically with other modules is making a | ||
| 17 | // combined work based on Avisynth. Thus, the terms and conditions of the GNU | ||
| 18 | // General Public License cover the whole combination. | ||
| 19 | // | ||
| 20 | // As a special exception, the copyright holders of Avisynth give you | ||
| 21 | // permission to link Avisynth with independent modules that communicate with | ||
| 22 | // Avisynth solely through the interfaces defined in avisynth.h, regardless of the license | ||
| 23 | // terms of these independent modules, and to copy and distribute the | ||
| 24 | // resulting combined work under terms of your choice, provided that | ||
| 25 | // every copy of the combined work is accompanied by a complete copy of | ||
| 26 | // the source code of Avisynth (the version of Avisynth used to produce the | ||
| 27 | // combined work), being distributed under the terms of the GNU General | ||
| 28 | // Public License plus this exception. An independent module is a module | ||
| 29 | // which is not derived from or based on Avisynth, such as 3rd-party filters, | ||
| 30 | // import and export plugins, or graphical user interfaces. | ||
| 31 | |||
| 32 | #include "bitblt.h" | ||
| 33 | #include <avs/config.h> | ||
| 34 | #include "memcpy_amd.h" | ||
| 35 | #include <avs/cpuid.h> | ||
| 36 | #include <cstring> | ||
| 37 | #include <cassert> | ||
| 38 | |||
| 39 | #ifdef INTEL_INTRINSICS | ||
| 40 | #if defined(X86_32) && defined(MSVC_PURE) | ||
| 41 | |||
| 42 | // Assembler bitblit by Steady | ||
| 43 | static void asm_BitBlt_ISSE(BYTE* dstp, int dst_pitch, const BYTE* srcp, int src_pitch, int row_size, int height) { | ||
| 44 | |||
| 45 | // Warning! : If you modify this routine, check the generated assembler to make sure | ||
| 46 | // the stupid compiler is saving the ebx register in the entry prologue. | ||
| 47 | // And don't just add an extra push/pop ebx pair around the code, try to | ||
| 48 | // convince the compiler to do the right thing, it's not hard, usually a | ||
| 49 | // slight shuffle or a well placed "__asm mov ebx,ebx" does the trick. | ||
| 50 | |||
| 51 | if(row_size==0 || height==0) return; //abort on goofs | ||
| 52 | //move backwards for easier looping and to disable hardware prefetch | ||
| 53 | const BYTE* srcStart=srcp+src_pitch*(height-1); | ||
| 54 | BYTE* dstStart=dstp+dst_pitch*(height-1); | ||
| 55 | |||
| 56 | if(row_size < 64) { | ||
| 57 | _asm { | ||
| 58 | mov esi,srcStart //move rows from bottom up | ||
| 59 | mov edi,dstStart | ||
| 60 | mov edx,row_size | ||
| 61 | dec edx | ||
| 62 | mov ebx,height | ||
| 63 | align 16 | ||
| 64 | memoptS_rowloop: | ||
| 65 | mov ecx,edx | ||
| 66 | // rep movsb | ||
| 67 | memoptS_byteloop: | ||
| 68 | mov AL,[esi+ecx] | ||
| 69 | mov [edi+ecx],AL | ||
| 70 | sub ecx,1 | ||
| 71 | jnc memoptS_byteloop | ||
| 72 | sub esi,src_pitch | ||
| 73 | sub edi,dst_pitch | ||
| 74 | dec ebx | ||
| 75 | jne memoptS_rowloop | ||
| 76 | }; | ||
| 77 | return; | ||
| 78 | }//end small version | ||
| 79 | |||
| 80 | else if( (int(dstp) | row_size | src_pitch | dst_pitch) & 7) {//not QW aligned | ||
| 81 | //unaligned version makes no assumptions on alignment | ||
| 82 | |||
| 83 | _asm { | ||
| 84 | //****** initialize | ||
| 85 | mov esi,srcStart //bottom row | ||
| 86 | mov AL,[esi] | ||
| 87 | mov edi,dstStart | ||
| 88 | mov edx,row_size | ||
| 89 | mov ebx,height | ||
| 90 | |||
| 91 | //********** loop starts here *********** | ||
| 92 | |||
| 93 | align 16 | ||
| 94 | memoptU_rowloop: | ||
| 95 | mov ecx,edx //row_size | ||
| 96 | dec ecx //offset to last byte in row | ||
| 97 | add ecx,esi //ecx= ptr last byte in row | ||
| 98 | and ecx,~63 //align to first byte in cache line | ||
| 99 | memoptU_prefetchloop: | ||
| 100 | mov AX,[ecx] //tried AL,AX,EAX, AX a tiny bit faster | ||
| 101 | sub ecx,64 | ||
| 102 | cmp ecx,esi | ||
| 103 | jae memoptU_prefetchloop | ||
| 104 | |||
| 105 | //************ write ************* | ||
| 106 | |||
| 107 | movq mm6,[esi] //move the first unaligned bytes | ||
| 108 | movntq [edi],mm6 | ||
| 109 | //************************ | ||
| 110 | mov eax,edi | ||
| 111 | neg eax | ||
| 112 | mov ecx,eax | ||
| 113 | and eax,63 //eax=bytes from [edi] to start of next 64 byte cache line | ||
| 114 | and ecx,7 //ecx=bytes from [edi] to next QW | ||
| 115 | align 16 | ||
| 116 | memoptU_prewrite8loop: //write out odd QW's so 64 bit write is cache line aligned | ||
| 117 | cmp ecx,eax //start of cache line ? | ||
| 118 | jz memoptU_pre8done //if not, write single QW | ||
| 119 | movq mm7,[esi+ecx] | ||
| 120 | movntq [edi+ecx],mm7 | ||
| 121 | add ecx,8 | ||
| 122 | jmp memoptU_prewrite8loop | ||
| 123 | |||
| 124 | align 16 | ||
| 125 | memoptU_write64loop: | ||
| 126 | movntq [edi+ecx-64],mm0 | ||
| 127 | movntq [edi+ecx-56],mm1 | ||
| 128 | movntq [edi+ecx-48],mm2 | ||
| 129 | movntq [edi+ecx-40],mm3 | ||
| 130 | movntq [edi+ecx-32],mm4 | ||
| 131 | movntq [edi+ecx-24],mm5 | ||
| 132 | movntq [edi+ecx-16],mm6 | ||
| 133 | movntq [edi+ecx- 8],mm7 | ||
| 134 | memoptU_pre8done: | ||
| 135 | add ecx,64 | ||
| 136 | cmp ecx,edx //while(offset <= row_size) do {... | ||
| 137 | ja memoptU_done64 | ||
| 138 | movq mm0,[esi+ecx-64] | ||
| 139 | movq mm1,[esi+ecx-56] | ||
| 140 | movq mm2,[esi+ecx-48] | ||
| 141 | movq mm3,[esi+ecx-40] | ||
| 142 | movq mm4,[esi+ecx-32] | ||
| 143 | movq mm5,[esi+ecx-24] | ||
| 144 | movq mm6,[esi+ecx-16] | ||
| 145 | movq mm7,[esi+ecx- 8] | ||
| 146 | jmp memoptU_write64loop | ||
| 147 | memoptU_done64: | ||
| 148 | |||
| 149 | sub ecx,64 //went to far | ||
| 150 | align 16 | ||
| 151 | memoptU_write8loop: | ||
| 152 | add ecx,8 //next QW | ||
| 153 | cmp ecx,edx //any QW's left in row ? | ||
| 154 | ja memoptU_done8 | ||
| 155 | movq mm0,[esi+ecx-8] | ||
| 156 | movntq [edi+ecx-8],mm0 | ||
| 157 | jmp memoptU_write8loop | ||
| 158 | memoptU_done8: | ||
| 159 | |||
| 160 | movq mm1,[esi+edx-8] //write the last unaligned bytes | ||
| 161 | movntq [edi+edx-8],mm1 | ||
| 162 | sub esi,src_pitch | ||
| 163 | sub edi,dst_pitch | ||
| 164 | dec ebx //row counter (=height at start) | ||
| 165 | jne memoptU_rowloop | ||
| 166 | |||
| 167 | sfence | ||
| 168 | emms | ||
| 169 | }; | ||
| 170 | return; | ||
| 171 | }//end unaligned version | ||
| 172 | |||
| 173 | else {//QW aligned version (fastest) | ||
| 174 | //else dstp and row_size QW aligned - hope for the best from srcp | ||
| 175 | //QW aligned version should generally be true when copying full rows | ||
| 176 | _asm { | ||
| 177 | mov esi,srcStart //start of bottom row | ||
| 178 | mov edi,dstStart | ||
| 179 | mov ebx,height | ||
| 180 | mov edx,row_size | ||
| 181 | align 16 | ||
| 182 | memoptA_rowloop: | ||
| 183 | mov ecx,edx //row_size | ||
| 184 | dec ecx //offset to last byte in row | ||
| 185 | |||
| 186 | //********forward routine | ||
| 187 | add ecx,esi | ||
| 188 | and ecx,~63 //align prefetch to first byte in cache line(~3-4% faster) | ||
| 189 | align 16 | ||
| 190 | memoptA_prefetchloop: | ||
| 191 | mov AX,[ecx] | ||
| 192 | sub ecx,64 | ||
| 193 | cmp ecx,esi | ||
| 194 | jae memoptA_prefetchloop | ||
| 195 | |||
| 196 | mov eax,edi | ||
| 197 | xor ecx,ecx | ||
| 198 | neg eax | ||
| 199 | and eax,63 //eax=bytes from edi to start of cache line | ||
| 200 | align 16 | ||
| 201 | memoptA_prewrite8loop: //write out odd QW's so 64bit write is cache line aligned | ||
| 202 | cmp ecx,eax //start of cache line ? | ||
| 203 | jz memoptA_pre8done //if not, write single QW | ||
| 204 | movq mm7,[esi+ecx] | ||
| 205 | movntq [edi+ecx],mm7 | ||
| 206 | add ecx,8 | ||
| 207 | jmp memoptA_prewrite8loop | ||
| 208 | |||
| 209 | align 16 | ||
| 210 | memoptA_write64loop: | ||
| 211 | movntq [edi+ecx-64],mm0 | ||
| 212 | movntq [edi+ecx-56],mm1 | ||
| 213 | movntq [edi+ecx-48],mm2 | ||
| 214 | movntq [edi+ecx-40],mm3 | ||
| 215 | movntq [edi+ecx-32],mm4 | ||
| 216 | movntq [edi+ecx-24],mm5 | ||
| 217 | movntq [edi+ecx-16],mm6 | ||
| 218 | movntq [edi+ecx- 8],mm7 | ||
| 219 | memoptA_pre8done: | ||
| 220 | add ecx,64 | ||
| 221 | cmp ecx,edx | ||
| 222 | ja memoptA_done64 //less than 64 bytes left | ||
| 223 | movq mm0,[esi+ecx-64] | ||
| 224 | movq mm1,[esi+ecx-56] | ||
| 225 | movq mm2,[esi+ecx-48] | ||
| 226 | movq mm3,[esi+ecx-40] | ||
| 227 | movq mm4,[esi+ecx-32] | ||
| 228 | movq mm5,[esi+ecx-24] | ||
| 229 | movq mm6,[esi+ecx-16] | ||
| 230 | movq mm7,[esi+ecx- 8] | ||
| 231 | jmp memoptA_write64loop | ||
| 232 | |||
| 233 | memoptA_done64: | ||
| 234 | sub ecx,64 | ||
| 235 | |||
| 236 | align 16 | ||
| 237 | memoptA_write8loop: //less than 8 QW's left | ||
| 238 | add ecx,8 | ||
| 239 | cmp ecx,edx | ||
| 240 | ja memoptA_done8 //no QW's left | ||
| 241 | movq mm7,[esi+ecx-8] | ||
| 242 | movntq [edi+ecx-8],mm7 | ||
| 243 | jmp memoptA_write8loop | ||
| 244 | |||
| 245 | memoptA_done8: | ||
| 246 | sub esi,src_pitch | ||
| 247 | sub edi,dst_pitch | ||
| 248 | dec ebx //row counter (height) | ||
| 249 | jne memoptA_rowloop | ||
| 250 | |||
| 251 | sfence | ||
| 252 | emms | ||
| 253 | }; | ||
| 254 | return; | ||
| 255 | }//end aligned version | ||
| 256 | }//end BitBlt_memopt() | ||
| 257 | |||
| 258 | #endif //X86_32 | ||
| 259 | #endif // INTEL_INTRINSICS | ||
| 260 | |||
| 261 | 540 | void BitBlt(BYTE* dstp, int dst_pitch, const BYTE* srcp, int src_pitch, int row_size, int height) | |
| 262 | { | ||
| 263 |
4/4✓ Branch 2 → 3 taken 473 times.
✓ Branch 2 → 4 taken 67 times.
✓ Branch 3 → 4 taken 1 time.
✓ Branch 3 → 5 taken 472 times.
|
540 | if ( (!height) || (!row_size) ) return; |
| 264 | |||
| 265 | #ifdef INTEL_INTRINSICS | ||
| 266 | #if defined(X86_32) && defined(MSVC_PURE) | ||
| 267 | const int cpuf = GetCPUFlags(); | ||
| 268 | if ((cpuf & CPUF_INTEGER_SSE) && !(cpuf & CPUF_AVX)) | ||
| 269 | { | ||
| 270 | if (height == 1 || (src_pitch == dst_pitch && dst_pitch == row_size)) { | ||
| 271 | memcpy_amd(dstp, srcp, row_size*height); | ||
| 272 | } else { | ||
| 273 | asm_BitBlt_ISSE(dstp,dst_pitch,srcp,src_pitch,row_size,height); | ||
| 274 | } | ||
| 275 | return; | ||
| 276 | } | ||
| 277 | #endif | ||
| 278 | #endif // INTEL_INTRINSICS | ||
| 279 | |||
| 280 |
6/6✓ Branch 5 → 6 taken 471 times.
✓ Branch 5 → 8 taken 1 time.
✓ Branch 6 → 7 taken 453 times.
✓ Branch 6 → 9 taken 18 times.
✓ Branch 7 → 8 taken 5 times.
✓ Branch 7 → 9 taken 448 times.
|
472 | if (height == 1 || (dst_pitch == src_pitch && src_pitch == row_size)) { |
| 281 | 6 | memcpy(dstp, srcp, row_size*height); | |
| 282 | } else { | ||
| 283 |
2/2✓ Branch 11 → 10 taken 1395 times.
✓ Branch 11 → 12 taken 466 times.
|
1861 | for (int y = height; y > 0; --y) { |
| 284 | 1395 | memcpy(dstp, srcp, row_size); | |
| 285 | 1395 | dstp += dst_pitch; | |
| 286 | 1395 | srcp += src_pitch; | |
| 287 | } | ||
| 288 | } | ||
| 289 | } | ||
| 290 |