|
1234567891011121314151617181920212223242526272829303132333435363738394041424344454647484950515253545556575859606162636465666768697071727374757677787980818283848586878889909192939495969798991001011021031041051061071081091101111121131141151161171181191201211221231241251261271281291301311321331341351361371381391401411421431441451461471481491501511521531541551561571581591601611621631641651661671681691701711721731741751761771781791801811821831841851861871881891901911921931941951961971981992002012022032042052062072082092102112122132142152162172182192202212222232242252262272282292302312322332342352362372382392402412422432442452462472482492502512522532542552562572582592602612622632642652662672682692702712722732742752762772782792802812822832842852862872882892902912922932942952962972982993003013023033043053063073083093103113123133143153163173183193203213223233243253263273283293303313323333343353363373383393403413423433443453463473483493503513523533543553563573583593603613623633643653663673683693703713723733743753763773783793803813823833843853863873883893903913923933943953963973983994004014024034044054064074084094104114124134144154164174184194204214224234244254264274284294304314324334344354364374384394404414424434444454464474484494504514524534544554564574584594604614624634644654664674684694704714724734744754764774784794804814824834844854864874884894904914924934944954964974984995005015025035045055065075085095105115125135145155165175185195205215225235245255265275285295305315325335345355365375385395405415425435445455465475485495505515525535545555565575585595605615625635645655665675685695705715725735745755765775785795805815825835845855865875885895905915925935945955965975985996006016026036046056066076086096106116126136146156166176186196206216226236246256266276286296306316326336346356366376386396406416426436446456466476486496506516526536546556566576586596606616626636646656666676686696706716726736746756766776786796806816826836846856866876886896906916926936946956966976986997007017027037047057067077087097107117127137147157167177187197207217227237247257267277287297307317327337347357367377387397407417427437447457467477487497507517527537547557567577587597607617627637647657667677687697707717727737747757767777787797807817827837847857867877887897907917927937947957967977987998008018028038048058068078088098108118128138148158168178188198208218228238248258268278288298308318328338348358368378388398408418428438448458468478488498508518528538548558568578588598608618628638648658668678688698708718728738748758768778788798808818828838848858868878888898908918928938948958968978988999009019029039049059069079089099109119129139149159169179189199209219229239249259269279289299309319329339349359369379389399409419429439449459469479489499509519529539549559569579589599609619629639649659669679689699709719729739749759769779789799809819829839849859869879889899909919929939949959969979989991000100110021003100410051006100710081009101010111012101310141015101610171018101910201021102210231024102510261027102810291030103110321033103410351036103710381039104010411042104310441045104610471048104910501051105210531054105510561057105810591060106110621063106410651066106710681069107010711072107310741075107610771078107910801081108210831084108510861087108810891090109110921093109410951096109710981099110011011102110311041105110611071108110911101111111211131114111511161117111811191120112111221123112411251126112711281129113011311132113311341135113611371138113911401141114211431144114511461147114811491150115111521153115411551156115711581159116011611162116311641165116611671168116911701171117211731174117511761177117811791180118111821183118411851186118711881189119011911192119311941195119611971198119912001201120212031204120512061207120812091210121112121213121412151216121712181219122012211222122312241225122612271228122912301231123212331234123512361237123812391240124112421243124412451246124712481249125012511252125312541255125612571258125912601261126212631264126512661267126812691270127112721273127412751276127712781279128012811282128312841285128612871288128912901291129212931294129512961297129812991300130113021303130413051306130713081309131013111312131313141315131613171318131913201321132213231324132513261327132813291330133113321333133413351336133713381339134013411342134313441345134613471348134913501351135213531354135513561357135813591360136113621363136413651366136713681369137013711372137313741375137613771378137913801381138213831384138513861387138813891390139113921393139413951396139713981399140014011402140314041405140614071408140914101411141214131414141514161417141814191420142114221423142414251426142714281429143014311432143314341435143614371438143914401441144214431444144514461447144814491450145114521453145414551456145714581459146014611462146314641465146614671468146914701471147214731474147514761477147814791480148114821483148414851486148714881489149014911492149314941495149614971498149915001501150215031504150515061507150815091510151115121513151415151516151715181519152015211522152315241525152615271528152915301531153215331534153515361537153815391540154115421543154415451546154715481549155015511552155315541555155615571558155915601561156215631564156515661567156815691570157115721573157415751576157715781579158015811582158315841585158615871588158915901591159215931594159515961597159815991600160116021603160416051606160716081609161016111612161316141615161616171618161916201621162216231624162516261627162816291630163116321633163416351636163716381639164016411642164316441645164616471648164916501651165216531654165516561657165816591660166116621663166416651666166716681669167016711672167316741675167616771678167916801681168216831684168516861687168816891690169116921693169416951696169716981699170017011702170317041705170617071708170917101711171217131714171517161717171817191720172117221723172417251726172717281729173017311732173317341735173617371738173917401741174217431744174517461747174817491750175117521753175417551756175717581759176017611762176317641765176617671768176917701771177217731774177517761777177817791780178117821783178417851786178717881789179017911792179317941795179617971798179918001801180218031804180518061807180818091810181118121813181418151816181718181819182018211822182318241825182618271828182918301831183218331834183518361837183818391840184118421843184418451846184718481849185018511852185318541855185618571858185918601861186218631864186518661867186818691870187118721873187418751876187718781879188018811882188318841885188618871888188918901891189218931894189518961897189818991900190119021903190419051906190719081909191019111912191319141915191619171918191919201921192219231924192519261927192819291930193119321933193419351936193719381939194019411942194319441945194619471948194919501951195219531954195519561957195819591960196119621963196419651966196719681969197019711972197319741975197619771978197919801981198219831984198519861987198819891990199119921993199419951996199719981999200020012002200320042005200620072008200920102011201220132014201520162017201820192020202120222023202420252026202720282029203020312032203320342035203620372038203920402041204220432044204520462047204820492050205120522053205420552056205720582059206020612062206320642065206620672068206920702071207220732074207520762077207820792080208120822083208420852086208720882089209020912092209320942095209620972098209921002101210221032104210521062107210821092110211121122113211421152116211721182119212021212122212321242125212621272128212921302131213221332134213521362137213821392140214121422143214421452146214721482149215021512152215321542155215621572158215921602161216221632164216521662167216821692170217121722173217421752176217721782179218021812182218321842185218621872188218921902191219221932194219521962197219821992200220122022203220422052206220722082209221022112212221322142215221622172218221922202221222222232224222522262227222822292230223122322233223422352236223722382239224022412242224322442245224622472248224922502251225222532254225522562257225822592260226122622263226422652266226722682269227022712272227322742275227622772278227922802281228222832284228522862287228822892290229122922293229422952296229722982299230023012302230323042305230623072308230923102311231223132314231523162317231823192320232123222323232423252326232723282329233023312332233323342335233623372338233923402341234223432344234523462347234823492350235123522353235423552356235723582359236023612362236323642365236623672368236923702371237223732374237523762377237823792380238123822383238423852386238723882389239023912392239323942395239623972398239924002401240224032404240524062407240824092410241124122413241424152416241724182419242024212422242324242425242624272428242924302431243224332434243524362437243824392440244124422443244424452446244724482449245024512452245324542455245624572458245924602461246224632464246524662467246824692470247124722473247424752476247724782479248024812482248324842485248624872488248924902491249224932494249524962497249824992500250125022503250425052506250725082509251025112512251325142515251625172518251925202521252225232524252525262527252825292530253125322533253425352536253725382539254025412542254325442545254625472548254925502551255225532554255525562557255825592560256125622563256425652566256725682569257025712572257325742575257625772578257925802581258225832584258525862587258825892590259125922593259425952596259725982599260026012602260326042605260626072608260926102611261226132614261526162617261826192620262126222623262426252626262726282629263026312632263326342635263626372638263926402641264226432644264526462647264826492650265126522653265426552656265726582659266026612662266326642665266626672668266926702671267226732674267526762677267826792680268126822683268426852686268726882689269026912692269326942695269626972698269927002701270227032704270527062707270827092710271127122713271427152716271727182719272027212722272327242725272627272728272927302731273227332734273527362737273827392740274127422743274427452746274727482749275027512752275327542755275627572758275927602761276227632764276527662767276827692770277127722773277427752776277727782779278027812782278327842785278627872788278927902791279227932794279527962797279827992800280128022803280428052806280728082809281028112812281328142815281628172818281928202821282228232824282528262827282828292830283128322833283428352836283728382839284028412842284328442845284628472848284928502851285228532854285528562857285828592860286128622863286428652866286728682869287028712872287328742875287628772878287928802881288228832884288528862887288828892890289128922893289428952896289728982899290029012902290329042905290629072908290929102911291229132914291529162917291829192920292129222923292429252926292729282929293029312932293329342935293629372938293929402941294229432944294529462947294829492950295129522953295429552956295729582959296029612962296329642965296629672968296929702971297229732974297529762977297829792980298129822983298429852986298729882989299029912992299329942995299629972998299930003001300230033004300530063007300830093010301130123013301430153016301730183019302030213022302330243025302630273028302930303031303230333034303530363037303830393040304130423043304430453046304730483049305030513052305330543055305630573058305930603061306230633064306530663067306830693070307130723073307430753076307730783079308030813082308330843085308630873088308930903091309230933094309530963097309830993100310131023103310431053106310731083109311031113112311331143115311631173118311931203121312231233124312531263127312831293130313131323133313431353136313731383139314031413142314331443145314631473148314931503151315231533154315531563157315831593160316131623163316431653166316731683169317031713172317331743175317631773178317931803181318231833184318531863187318831893190319131923193319431953196319731983199320032013202320332043205320632073208320932103211321232133214321532163217321832193220322132223223322432253226322732283229323032313232323332343235323632373238323932403241324232433244324532463247324832493250325132523253325432553256325732583259326032613262326332643265326632673268326932703271327232733274327532763277327832793280328132823283328432853286328732883289329032913292329332943295329632973298329933003301330233033304330533063307330833093310331133123313331433153316331733183319332033213322332333243325332633273328332933303331333233333334333533363337333833393340334133423343334433453346334733483349335033513352335333543355335633573358335933603361336233633364336533663367336833693370337133723373337433753376337733783379338033813382338333843385338633873388338933903391339233933394339533963397339833993400340134023403340434053406340734083409341034113412341334143415341634173418341934203421342234233424342534263427342834293430343134323433343434353436343734383439344034413442344334443445344634473448344934503451345234533454345534563457345834593460346134623463346434653466346734683469347034713472347334743475347634773478347934803481348234833484348534863487348834893490349134923493349434953496349734983499350035013502350335043505350635073508350935103511351235133514351535163517351835193520352135223523352435253526352735283529353035313532353335343535353635373538353935403541354235433544354535463547354835493550355135523553355435553556355735583559356035613562356335643565356635673568356935703571357235733574357535763577357835793580358135823583358435853586358735883589359035913592359335943595359635973598359936003601360236033604360536063607360836093610361136123613361436153616361736183619362036213622362336243625362636273628362936303631363236333634363536363637363836393640364136423643364436453646364736483649365036513652365336543655365636573658365936603661366236633664366536663667366836693670367136723673367436753676367736783679368036813682368336843685368636873688368936903691369236933694369536963697369836993700370137023703370437053706370737083709371037113712371337143715371637173718371937203721372237233724372537263727372837293730373137323733373437353736373737383739374037413742374337443745374637473748374937503751375237533754375537563757375837593760376137623763376437653766376737683769377037713772377337743775377637773778377937803781378237833784378537863787378837893790379137923793379437953796379737983799380038013802380338043805380638073808380938103811381238133814381538163817381838193820382138223823382438253826382738283829383038313832383338343835383638373838383938403841384238433844384538463847384838493850385138523853385438553856385738583859386038613862386338643865386638673868386938703871387238733874387538763877387838793880388138823883388438853886388738883889389038913892389338943895389638973898389939003901390239033904390539063907390839093910391139123913391439153916391739183919392039213922392339243925392639273928392939303931393239333934393539363937393839393940394139423943394439453946394739483949395039513952395339543955395639573958395939603961396239633964396539663967396839693970397139723973397439753976397739783979398039813982398339843985398639873988398939903991399239933994399539963997399839994000400140024003400440054006400740084009401040114012401340144015401640174018401940204021402240234024402540264027402840294030403140324033403440354036403740384039404040414042404340444045404640474048404940504051405240534054405540564057405840594060406140624063406440654066406740684069407040714072407340744075407640774078407940804081408240834084408540864087408840894090409140924093409440954096409740984099410041014102410341044105410641074108410941104111411241134114411541164117411841194120412141224123412441254126412741284129413041314132413341344135413641374138413941404141414241434144414541464147414841494150415141524153415441554156415741584159416041614162416341644165416641674168416941704171417241734174417541764177417841794180418141824183418441854186418741884189419041914192419341944195419641974198419942004201420242034204420542064207420842094210421142124213421442154216421742184219422042214222422342244225422642274228422942304231423242334234423542364237423842394240424142424243424442454246424742484249425042514252425342544255425642574258425942604261426242634264426542664267426842694270427142724273427442754276427742784279428042814282428342844285428642874288428942904291429242934294429542964297429842994300430143024303430443054306430743084309431043114312431343144315431643174318431943204321432243234324432543264327432843294330433143324333433443354336433743384339434043414342434343444345434643474348434943504351435243534354435543564357435843594360436143624363436443654366436743684369437043714372437343744375437643774378437943804381438243834384438543864387438843894390439143924393439443954396439743984399440044014402440344044405440644074408440944104411441244134414441544164417441844194420442144224423442444254426442744284429443044314432443344344435443644374438443944404441444244434444444544464447444844494450445144524453445444554456445744584459446044614462446344644465446644674468446944704471447244734474447544764477447844794480448144824483448444854486448744884489449044914492449344944495449644974498449945004501450245034504450545064507450845094510451145124513451445154516451745184519452045214522452345244525452645274528452945304531453245334534453545364537453845394540454145424543454445454546454745484549455045514552455345544555455645574558455945604561456245634564456545664567456845694570457145724573457445754576457745784579458045814582458345844585458645874588458945904591459245934594459545964597459845994600460146024603460446054606460746084609461046114612461346144615461646174618461946204621462246234624462546264627462846294630463146324633463446354636463746384639464046414642464346444645464646474648464946504651465246534654465546564657465846594660466146624663466446654666466746684669467046714672467346744675467646774678467946804681468246834684468546864687468846894690469146924693469446954696469746984699470047014702470347044705470647074708470947104711471247134714471547164717471847194720472147224723472447254726472747284729473047314732473347344735473647374738473947404741474247434744474547464747474847494750475147524753475447554756475747584759476047614762476347644765476647674768476947704771477247734774477547764777477847794780478147824783478447854786478747884789479047914792479347944795479647974798479948004801480248034804480548064807480848094810481148124813481448154816481748184819482048214822482348244825482648274828482948304831483248334834483548364837483848394840484148424843484448454846484748484849485048514852485348544855485648574858485948604861486248634864486548664867486848694870487148724873487448754876487748784879488048814882488348844885488648874888488948904891489248934894489548964897489848994900490149024903490449054906490749084909491049114912491349144915491649174918491949204921492249234924492549264927492849294930493149324933493449354936493749384939494049414942494349444945494649474948494949504951495249534954495549564957495849594960496149624963496449654966496749684969497049714972497349744975497649774978497949804981498249834984498549864987498849894990499149924993499449954996499749984999500050015002500350045005500650075008500950105011501250135014501550165017501850195020502150225023502450255026502750285029503050315032503350345035503650375038503950405041504250435044504550465047504850495050505150525053505450555056505750585059506050615062506350645065506650675068506950705071507250735074507550765077507850795080508150825083508450855086508750885089509050915092509350945095509650975098509951005101510251035104510551065107510851095110511151125113511451155116511751185119512051215122512351245125512651275128512951305131513251335134513551365137513851395140514151425143514451455146514751485149515051515152515351545155515651575158515951605161516251635164516551665167516851695170517151725173517451755176517751785179518051815182518351845185518651875188518951905191519251935194519551965197519851995200520152025203520452055206520752085209521052115212521352145215521652175218521952205221522252235224522552265227522852295230523152325233523452355236523752385239524052415242524352445245524652475248524952505251525252535254525552565257525852595260526152625263526452655266526752685269527052715272527352745275527652775278527952805281528252835284528552865287528852895290529152925293529452955296529752985299530053015302530353045305530653075308530953105311531253135314531553165317531853195320532153225323532453255326532753285329533053315332533353345335533653375338533953405341534253435344534553465347534853495350535153525353535453555356535753585359536053615362536353645365536653675368536953705371537253735374537553765377537853795380538153825383538453855386538753885389539053915392539353945395539653975398539954005401540254035404540554065407540854095410541154125413541454155416541754185419542054215422542354245425542654275428542954305431543254335434543554365437543854395440544154425443544454455446544754485449545054515452545354545455545654575458545954605461546254635464546554665467546854695470547154725473547454755476547754785479548054815482548354845485548654875488548954905491549254935494549554965497549854995500550155025503550455055506550755085509551055115512551355145515551655175518551955205521552255235524552555265527552855295530553155325533553455355536553755385539554055415542554355445545554655475548554955505551555255535554555555565557555855595560556155625563556455655566556755685569557055715572557355745575557655775578557955805581558255835584558555865587558855895590559155925593559455955596559755985599560056015602560356045605560656075608560956105611561256135614561556165617561856195620562156225623562456255626562756285629563056315632563356345635563656375638563956405641564256435644564556465647564856495650565156525653565456555656565756585659566056615662566356645665566656675668566956705671567256735674567556765677567856795680568156825683568456855686568756885689569056915692569356945695569656975698569957005701570257035704570557065707570857095710571157125713571457155716571757185719572057215722572357245725572657275728572957305731573257335734573557365737573857395740574157425743574457455746574757485749575057515752575357545755575657575758575957605761576257635764576557665767576857695770577157725773577457755776577757785779578057815782578357845785578657875788578957905791579257935794579557965797579857995800580158025803580458055806580758085809581058115812581358145815581658175818581958205821582258235824582558265827582858295830583158325833583458355836583758385839584058415842584358445845584658475848584958505851585258535854585558565857585858595860586158625863586458655866586758685869587058715872587358745875587658775878587958805881588258835884588558865887588858895890589158925893589458955896589758985899590059015902590359045905590659075908590959105911591259135914591559165917591859195920592159225923592459255926592759285929593059315932593359345935593659375938593959405941594259435944594559465947594859495950595159525953595459555956595759585959596059615962596359645965596659675968596959705971597259735974597559765977597859795980598159825983598459855986598759885989599059915992599359945995599659975998599960006001600260036004600560066007600860096010601160126013601460156016601760186019602060216022602360246025602660276028602960306031603260336034603560366037603860396040604160426043604460456046604760486049605060516052605360546055605660576058605960606061606260636064606560666067606860696070607160726073607460756076607760786079608060816082608360846085608660876088608960906091609260936094609560966097609860996100610161026103610461056106 |
- // Copyright 2019 Tencent
- // SPDX-License-Identifier: BSD-3-Clause
-
- #include "mat.h"
- #if __ARM_NEON
- #include <arm_neon.h>
- #endif // __ARM_NEON
- #include "platform.h"
-
- namespace ncnn {
-
- #if NCNN_PIXEL_ROTATE
- // should be a kanna ascii art here in my local branch
- // but we shall ask the original art author for permission first ...
- // https://www.reddit.com/r/anime/comments/5uxjn4/i_recreated_the_kanna_ascii_art_from_kobayashisan/
-
- static void kanna_rotate_1_c1(const unsigned char* src, int srcw, int srch, int srcstride, unsigned char* dst, int w, int /*h*/, int stride)
- {
- const int srcwgap = srcstride - srcw;
- const int wgap = stride - w;
-
- const unsigned char* src0 = src;
- const unsigned char* src1 = src + srcstride;
- unsigned char* dst0 = dst;
- unsigned char* dst1 = dst + stride;
-
- int y = 0;
- for (; y + 1 < srch; y += 2)
- {
- #if __ARM_NEON
- int nn = srcw >> 5;
- int remain = srcw - (nn << 5);
- #if !NCNN_GNU_INLINE_ASM || __aarch64__
- for (; nn > 0; nn--)
- {
- uint8x16_t _src0 = vld1q_u8(src0);
- uint8x16_t _src0n = vld1q_u8(src0 + 16);
- vst1q_u8(dst0, _src0);
- vst1q_u8(dst0 + 16, _src0n);
-
- uint8x16_t _src1 = vld1q_u8(src1);
- uint8x16_t _src1n = vld1q_u8(src1 + 16);
- vst1q_u8(dst1, _src1);
- vst1q_u8(dst1 + 16, _src1n);
-
- src0 += 32;
- src1 += 32;
- dst0 += 32;
- dst1 += 32;
- }
- #else
- if (nn > 0)
- {
- asm volatile(
- "0: \n"
- "pld [%1, #256] \n"
- "vld1.u8 {d0-d3}, [%1]! \n"
- "pld [%2, #256] \n"
- "vld1.u8 {d4-d7}, [%2]! \n"
- "subs %0, #1 \n"
- "vst1.u8 {d0-d3}, [%3]! \n"
- "vst1.u8 {d4-d7}, [%4]! \n"
- "bne 0b \n"
- : "=r"(nn), // %0
- "=r"(src0), // %1
- "=r"(src1), // %2
- "=r"(dst0), // %3
- "=r"(dst1) // %4
- : "0"(nn),
- "1"(src0),
- "2"(src1),
- "3"(dst0),
- "4"(dst1)
- : "cc", "memory", "q0", "q1", "q2", "q3");
- }
- #endif // __aarch64__
- #else
- int remain = srcw;
- #endif // __ARM_NEON
-
- for (; remain > 0; remain--)
- {
- *dst0++ = *src0++;
- *dst1++ = *src1++;
- }
-
- src0 += srcwgap + srcstride;
- src1 += srcwgap + srcstride;
- dst0 += wgap + stride;
- dst1 += wgap + stride;
- }
-
- for (; y < srch; y++)
- {
- #if __ARM_NEON
- int nn = srcw >> 5;
- int remain = srcw - (nn << 5);
- #if !NCNN_GNU_INLINE_ASM || __aarch64__
- for (; nn > 0; nn--)
- {
- uint8x16_t _src = vld1q_u8(src0);
- uint8x16_t _src2 = vld1q_u8(src0 + 16);
- vst1q_u8(dst0, _src);
- vst1q_u8(dst0 + 16, _src2);
-
- src0 += 32;
- dst0 += 32;
- }
- #else
- if (nn > 0)
- {
- asm volatile(
- "0: \n"
- "pld [%1, #256] \n"
- "vld1.u8 {d0-d3}, [%1]! \n"
- "subs %0, #1 \n"
- "vst1.u8 {d0-d3}, [%2]! \n"
- "bne 0b \n"
- : "=r"(nn), // %0
- "=r"(src0), // %1
- "=r"(dst0) // %2
- : "0"(nn),
- "1"(src0),
- "2"(dst0)
- : "cc", "memory", "q0", "q1");
- }
- #endif // __aarch64__
- #else
- int remain = srcw;
- #endif // __ARM_NEON
-
- for (; remain > 0; remain--)
- {
- *dst0++ = *src0++;
- }
-
- src0 += srcwgap;
- dst0 += wgap;
- }
- }
-
- static void kanna_rotate_1_c2(const unsigned char* src, int srcw, int srch, int srcstride, unsigned char* dst, int w, int /*h*/, int stride)
- {
- const int srcwgap = srcstride - srcw * 2;
- const int wgap = stride - w * 2;
-
- int size = srcw * 2;
-
- const unsigned char* src0 = src;
- const unsigned char* src1 = src + srcstride;
- unsigned char* dst0 = dst;
- unsigned char* dst1 = dst + stride;
-
- int y = 0;
- for (; y + 1 < srch; y += 2)
- {
- #if __ARM_NEON
- int nn = size >> 5;
- int remain = size - (nn << 5);
- #if !NCNN_GNU_INLINE_ASM || __aarch64__
- for (; nn > 0; nn--)
- {
- uint8x16_t _src0 = vld1q_u8(src0);
- uint8x16_t _src0n = vld1q_u8(src0 + 16);
- vst1q_u8(dst0, _src0);
- vst1q_u8(dst0 + 16, _src0n);
-
- uint8x16_t _src1 = vld1q_u8(src1);
- uint8x16_t _src1n = vld1q_u8(src1 + 16);
- vst1q_u8(dst1, _src1);
- vst1q_u8(dst1 + 16, _src1n);
-
- src0 += 32;
- src1 += 32;
- dst0 += 32;
- dst1 += 32;
- }
- #else
- if (nn > 0)
- {
- asm volatile(
- "0: \n"
- "pld [%1, #256] \n"
- "vld1.u8 {d0-d3}, [%1]! \n"
- "pld [%2, #256] \n"
- "vld1.u8 {d4-d7}, [%2]! \n"
- "subs %0, #1 \n"
- "vst1.u8 {d0-d3}, [%3]! \n"
- "vst1.u8 {d4-d7}, [%4]! \n"
- "bne 0b \n"
- : "=r"(nn), // %0
- "=r"(src0), // %1
- "=r"(src1), // %2
- "=r"(dst0), // %3
- "=r"(dst1) // %4
- : "0"(nn),
- "1"(src0),
- "2"(src1),
- "3"(dst0),
- "4"(dst1)
- : "cc", "memory", "q0", "q1", "q2", "q3");
- }
- #endif // __aarch64__
- #else
- int remain = size;
- #endif // __ARM_NEON
-
- for (; remain > 0; remain--)
- {
- *dst0++ = *src0++;
- *dst1++ = *src1++;
- }
-
- src0 += srcwgap + srcstride;
- src1 += srcwgap + srcstride;
- dst0 += wgap + stride;
- dst1 += wgap + stride;
- }
-
- for (; y < srch; y++)
- {
- #if __ARM_NEON
- int nn = size >> 5;
- int remain = size - (nn << 5);
- #if !NCNN_GNU_INLINE_ASM || __aarch64__
- for (; nn > 0; nn--)
- {
- uint8x16_t _src = vld1q_u8(src0);
- uint8x16_t _src2 = vld1q_u8(src0 + 16);
- vst1q_u8(dst0, _src);
- vst1q_u8(dst0 + 16, _src2);
-
- src0 += 32;
- dst0 += 32;
- }
- #else
- if (nn > 0)
- {
- asm volatile(
- "0: \n"
- "pld [%1, #256] \n"
- "vld1.u8 {d0-d3}, [%1]! \n"
- "subs %0, #1 \n"
- "vst1.u8 {d0-d3}, [%2]! \n"
- "bne 0b \n"
- : "=r"(nn), // %0
- "=r"(src0), // %1
- "=r"(dst0) // %2
- : "0"(nn),
- "1"(src0),
- "2"(dst0)
- : "cc", "memory", "q0", "q1");
- }
- #endif // __aarch64__
- #else
- int remain = size;
- #endif // __ARM_NEON
-
- for (; remain > 0; remain--)
- {
- *dst0++ = *src0++;
- }
-
- src0 += srcwgap;
- dst0 += wgap;
- }
- }
-
- static void kanna_rotate_1_c3(const unsigned char* src, int srcw, int srch, int srcstride, unsigned char* dst, int w, int /*h*/, int stride)
- {
- const int srcwgap = srcstride - srcw * 3;
- const int wgap = stride - w * 3;
-
- int size = srcw * 3;
-
- const unsigned char* src0 = src;
- const unsigned char* src1 = src + srcstride;
- unsigned char* dst0 = dst;
- unsigned char* dst1 = dst + stride;
-
- int y = 0;
- for (; y + 1 < srch; y += 2)
- {
- #if __ARM_NEON
- int nn = size >> 5;
- int remain = size - (nn << 5);
- #if !NCNN_GNU_INLINE_ASM || __aarch64__
- for (; nn > 0; nn--)
- {
- uint8x16_t _src0 = vld1q_u8(src0);
- uint8x16_t _src0n = vld1q_u8(src0 + 16);
- vst1q_u8(dst0, _src0);
- vst1q_u8(dst0 + 16, _src0n);
-
- uint8x16_t _src1 = vld1q_u8(src1);
- uint8x16_t _src1n = vld1q_u8(src1 + 16);
- vst1q_u8(dst1, _src1);
- vst1q_u8(dst1 + 16, _src1n);
-
- src0 += 32;
- src1 += 32;
- dst0 += 32;
- dst1 += 32;
- }
- #else
- if (nn > 0)
- {
- asm volatile(
- "0: \n"
- "pld [%1, #256] \n"
- "vld1.u8 {d0-d3}, [%1]! \n"
- "pld [%2, #256] \n"
- "vld1.u8 {d4-d7}, [%2]! \n"
- "subs %0, #1 \n"
- "vst1.u8 {d0-d3}, [%3]! \n"
- "vst1.u8 {d4-d7}, [%4]! \n"
- "bne 0b \n"
- : "=r"(nn), // %0
- "=r"(src0), // %1
- "=r"(src1), // %2
- "=r"(dst0), // %3
- "=r"(dst1) // %4
- : "0"(nn),
- "1"(src0),
- "2"(src1),
- "3"(dst0),
- "4"(dst1)
- : "cc", "memory", "q0", "q1", "q2", "q3");
- }
- #endif // __aarch64__
- #else
- int remain = size;
- #endif // __ARM_NEON
-
- for (; remain > 0; remain--)
- {
- *dst0++ = *src0++;
- *dst1++ = *src1++;
- }
-
- src0 += srcwgap + srcstride;
- src1 += srcwgap + srcstride;
- dst0 += wgap + stride;
- dst1 += wgap + stride;
- }
-
- for (; y < srch; y++)
- {
- #if __ARM_NEON
- int nn = size >> 5;
- int remain = size - (nn << 5);
- #if !NCNN_GNU_INLINE_ASM || __aarch64__
- for (; nn > 0; nn--)
- {
- uint8x16_t _src = vld1q_u8(src0);
- uint8x16_t _src2 = vld1q_u8(src0 + 16);
- vst1q_u8(dst0, _src);
- vst1q_u8(dst0 + 16, _src2);
-
- src0 += 32;
- dst0 += 32;
- }
- #else
- if (nn > 0)
- {
- asm volatile(
- "0: \n"
- "pld [%1, #256] \n"
- "vld1.u8 {d0-d3}, [%1]! \n"
- "subs %0, #1 \n"
- "vst1.u8 {d0-d3}, [%2]! \n"
- "bne 0b \n"
- : "=r"(nn), // %0
- "=r"(src0), // %1
- "=r"(dst0) // %2
- : "0"(nn),
- "1"(src0),
- "2"(dst0)
- : "cc", "memory", "q0", "q1");
- }
- #endif // __aarch64__
- #else
- int remain = size;
- #endif // __ARM_NEON
-
- for (; remain > 0; remain--)
- {
- *dst0++ = *src0++;
- }
-
- src0 += srcwgap;
- dst0 += wgap;
- }
- }
-
- static void kanna_rotate_1_c4(const unsigned char* src, int srcw, int srch, int srcstride, unsigned char* dst, int w, int /*h*/, int stride)
- {
- const int srcwgap = srcstride - srcw * 4;
- const int wgap = stride - w * 4;
-
- int size = srcw * 4;
-
- const unsigned char* src0 = src;
- const unsigned char* src1 = src + srcstride;
- unsigned char* dst0 = dst;
- unsigned char* dst1 = dst + stride;
-
- int y = 0;
- for (; y + 1 < srch; y += 2)
- {
- #if __ARM_NEON
- int nn = size >> 5;
- int remain = size - (nn << 5);
- #if !NCNN_GNU_INLINE_ASM || __aarch64__
- for (; nn > 0; nn--)
- {
- uint8x16_t _src0 = vld1q_u8(src0);
- uint8x16_t _src0n = vld1q_u8(src0 + 16);
- vst1q_u8(dst0, _src0);
- vst1q_u8(dst0 + 16, _src0n);
-
- uint8x16_t _src1 = vld1q_u8(src1);
- uint8x16_t _src1n = vld1q_u8(src1 + 16);
- vst1q_u8(dst1, _src1);
- vst1q_u8(dst1 + 16, _src1n);
-
- src0 += 32;
- src1 += 32;
- dst0 += 32;
- dst1 += 32;
- }
- #else
- if (nn > 0)
- {
- asm volatile(
- "0: \n"
- "pld [%1, #256] \n"
- "vld1.u8 {d0-d3}, [%1]! \n"
- "pld [%2, #256] \n"
- "vld1.u8 {d4-d7}, [%2]! \n"
- "subs %0, #1 \n"
- "vst1.u8 {d0-d3}, [%3]! \n"
- "vst1.u8 {d4-d7}, [%4]! \n"
- "bne 0b \n"
- : "=r"(nn), // %0
- "=r"(src0), // %1
- "=r"(src1), // %2
- "=r"(dst0), // %3
- "=r"(dst1) // %4
- : "0"(nn),
- "1"(src0),
- "2"(src1),
- "3"(dst0),
- "4"(dst1)
- : "cc", "memory", "q0", "q1", "q2", "q3");
- }
- #endif // __aarch64__
- #else
- int remain = size;
- #endif // __ARM_NEON
-
- for (; remain > 0; remain--)
- {
- *dst0++ = *src0++;
- *dst1++ = *src1++;
- }
-
- src0 += srcwgap + srcstride;
- src1 += srcwgap + srcstride;
- dst0 += wgap + stride;
- dst1 += wgap + stride;
- }
-
- for (; y < srch; y++)
- {
- #if __ARM_NEON
- int nn = size >> 5;
- int remain = size - (nn << 5);
- #if !NCNN_GNU_INLINE_ASM || __aarch64__
- for (; nn > 0; nn--)
- {
- uint8x16_t _src = vld1q_u8(src0);
- uint8x16_t _src2 = vld1q_u8(src0 + 16);
- vst1q_u8(dst0, _src);
- vst1q_u8(dst0 + 16, _src2);
-
- src0 += 32;
- dst0 += 32;
- }
- #else
- if (nn > 0)
- {
- asm volatile(
- "0: \n"
- "pld [%1, #256] \n"
- "vld1.u8 {d0-d3}, [%1]! \n"
- "subs %0, #1 \n"
- "vst1.u8 {d0-d3}, [%2]! \n"
- "bne 0b \n"
- : "=r"(nn), // %0
- "=r"(src0), // %1
- "=r"(dst0) // %2
- : "0"(nn),
- "1"(src0),
- "2"(dst0)
- : "cc", "memory", "q0", "q1");
- }
- #endif // __aarch64__
- #else
- int remain = size;
- #endif // __ARM_NEON
-
- for (; remain > 0; remain--)
- {
- *dst0++ = *src0++;
- }
-
- src0 += srcwgap;
- dst0 += wgap;
- }
- }
-
- static void kanna_rotate_2_c1(const unsigned char* src, int srcw, int srch, int srcstride, unsigned char* dst, int w, int /*h*/, int stride)
- {
- const int srcwgap = srcstride - srcw;
- const int wgap = stride + w;
-
- const unsigned char* src0 = src;
- unsigned char* dst0 = dst + w - 1;
-
- int y = 0;
- for (; y < srch; y++)
- {
- #if __ARM_NEON
- dst0 -= 15;
-
- int nn = srcw >> 4;
- int remain = srcw - (nn << 4);
-
- #if !NCNN_GNU_INLINE_ASM || __aarch64__
- for (; nn > 0; nn--)
- {
- uint8x8_t _src = vld1_u8(src0);
- uint8x8_t _src2 = vld1_u8(src0 + 8);
-
- _src = vrev64_u8(_src);
- _src2 = vrev64_u8(_src2);
-
- vst1_u8(dst0, _src2);
- vst1_u8(dst0 + 8, _src);
-
- src0 += 16;
- dst0 -= 16;
- }
- #else
- if (nn > 0)
- {
- asm volatile(
- "mov r4, #-16 \n"
- "0: \n"
- "pld [%1, #128] \n"
- "vld1.u8 {d0-d1}, [%1]! \n"
- "vrev64.u8 d3, d0 \n"
- "vrev64.u8 d2, d1 \n"
- "subs %0, #1 \n"
- "vst1.u8 {d2-d3}, [%2], r4 \n"
- "bne 0b \n"
- : "=r"(nn), // %0
- "=r"(src0), // %1
- "=r"(dst0) // %2
- : "0"(nn),
- "1"(src0),
- "2"(dst0)
- : "cc", "memory", "q0", "q1", "r4");
- }
- #endif // __aarch64__
-
- dst0 += 15;
- #else
- int remain = srcw;
- #endif // __ARM_NEON
-
- for (; remain > 0; remain--)
- {
- *dst0 = *src0;
-
- src0 += 1;
- dst0 -= 1;
- }
-
- src0 += srcwgap;
- dst0 += wgap;
- }
- }
-
- static void kanna_rotate_2_c2(const unsigned char* src, int srcw, int srch, int srcstride, unsigned char* dst, int w, int /*h*/, int stride)
- {
- const int srcwgap = srcstride - srcw * 2;
- const int wgap = stride + w * 2;
-
- const unsigned char* src0 = src;
- unsigned char* dst0 = dst + w * 2 - 2;
-
- int y = 0;
- for (; y < srch; y++)
- {
- #if __ARM_NEON
- dst0 -= 7 * 2;
-
- int nn = srcw >> 4;
- int remain = srcw - (nn << 4);
-
- #if !NCNN_GNU_INLINE_ASM || __aarch64__
- for (; nn > 0; nn--)
- {
- uint8x8x2_t _src = vld2_u8(src0);
- uint8x8x2_t _src2 = vld2_u8(src0 + 8 * 2);
-
- _src.val[0] = vrev64_u8(_src.val[0]);
- _src.val[1] = vrev64_u8(_src.val[1]);
-
- _src2.val[0] = vrev64_u8(_src2.val[0]);
- _src2.val[1] = vrev64_u8(_src2.val[1]);
-
- vst2_u8(dst0, _src);
- vst2_u8(dst0 - 8 * 2, _src2);
-
- src0 += 16 * 2;
- dst0 -= 16 * 2;
- }
- #else
- if (nn > 0)
- {
- asm volatile(
- "mov r4, #-16 \n"
- "0: \n"
- "pld [%1, #128] \n"
- "vld2.u8 {d0-d1}, [%1]! \n"
- "vrev64.u8 d0, d0 \n"
- "pld [%1, #128] \n"
- "vld2.u8 {d2-d3}, [%1]! \n"
- "vrev64.u8 d1, d1 \n"
- "vrev64.u8 d2, d2 \n"
- "vst2.u8 {d0-d1}, [%2], r4 \n"
- "vrev64.u8 d3, d3 \n"
- "subs %0, #1 \n"
- "vst2.u8 {d2-d3}, [%2], r4 \n"
- "bne 0b \n"
- : "=r"(nn), // %0
- "=r"(src0), // %1
- "=r"(dst0) // %2
- : "0"(nn),
- "1"(src0),
- "2"(dst0)
- : "cc", "memory", "q0", "q1", "r4");
- }
- #endif // __aarch64__
-
- dst0 += 7 * 2;
- #else
- int remain = srcw;
- #endif // __ARM_NEON
-
- for (; remain > 0; remain--)
- {
- dst0[0] = src0[0];
- dst0[1] = src0[1];
-
- src0 += 2;
- dst0 -= 2;
- }
-
- src0 += srcwgap;
- dst0 += wgap;
- }
- }
-
- static void kanna_rotate_2_c3(const unsigned char* src, int srcw, int srch, int srcstride, unsigned char* dst, int w, int /*h*/, int stride)
- {
- const int srcwgap = srcstride - srcw * 3;
- const int wgap = stride + w * 3;
-
- const unsigned char* src0 = src;
- unsigned char* dst0 = dst + w * 3 - 3;
-
- int y = 0;
- for (; y < srch; y++)
- {
- #if __ARM_NEON
- dst0 -= 7 * 3;
-
- int nn = srcw >> 4;
- int remain = srcw - (nn << 4);
-
- #if !NCNN_GNU_INLINE_ASM || __aarch64__
- for (; nn > 0; nn--)
- {
- uint8x8x3_t _src = vld3_u8(src0);
- uint8x8x3_t _src2 = vld3_u8(src0 + 8 * 3);
-
- _src.val[0] = vrev64_u8(_src.val[0]);
- _src.val[1] = vrev64_u8(_src.val[1]);
- _src.val[2] = vrev64_u8(_src.val[2]);
-
- _src2.val[0] = vrev64_u8(_src2.val[0]);
- _src2.val[1] = vrev64_u8(_src2.val[1]);
- _src2.val[2] = vrev64_u8(_src2.val[2]);
-
- vst3_u8(dst0, _src);
- vst3_u8(dst0 - 8 * 3, _src2);
-
- src0 += 16 * 3;
- dst0 -= 16 * 3;
- }
- #else
- if (nn > 0)
- {
- asm volatile(
- "mov r4, #-24 \n"
- "0: \n"
- "pld [%1, #192] \n"
- "vld3.u8 {d0-d2}, [%1]! \n"
- "vrev64.u8 d0, d0 \n"
- "vrev64.u8 d1, d1 \n"
- "pld [%1, #192] \n"
- "vld3.u8 {d4-d6}, [%1]! \n"
- "vrev64.u8 d2, d2 \n"
- "vrev64.u8 d4, d4 \n"
- "vst3.u8 {d0-d2}, [%2], r4 \n"
- "vrev64.u8 d5, d5 \n"
- "vrev64.u8 d6, d6 \n"
- "subs %0, #1 \n"
- "vst3.u8 {d4-d6}, [%2], r4 \n"
- "bne 0b \n"
- : "=r"(nn), // %0
- "=r"(src0), // %1
- "=r"(dst0) // %2
- : "0"(nn),
- "1"(src0),
- "2"(dst0)
- : "cc", "memory", "q0", "q1", "q2", "q3", "r4");
- }
- #endif // __aarch64__
-
- dst0 += 7 * 3;
- #else
- int remain = srcw;
- #endif // __ARM_NEON
-
- for (; remain > 0; remain--)
- {
- dst0[0] = src0[0];
- dst0[1] = src0[1];
- dst0[2] = src0[2];
-
- src0 += 3;
- dst0 -= 3;
- }
-
- src0 += srcwgap;
- dst0 += wgap;
- }
- }
-
- static void kanna_rotate_2_c4(const unsigned char* src, int srcw, int srch, int srcstride, unsigned char* dst, int w, int /*h*/, int stride)
- {
- const int srcwgap = srcstride - srcw * 4;
- const int wgap = stride + w * 4;
-
- const unsigned char* src0 = src;
- unsigned char* dst0 = dst + w * 4 - 4;
-
- int y = 0;
- for (; y < srch; y++)
- {
- #if __ARM_NEON
- dst0 -= 7 * 4;
-
- int nn = srcw >> 4;
- int remain = srcw - (nn << 4);
-
- #if !NCNN_GNU_INLINE_ASM || __aarch64__
- for (; nn > 0; nn--)
- {
- uint8x8x4_t _src = vld4_u8(src0);
- uint8x8x4_t _src2 = vld4_u8(src0 + 8 * 4);
-
- _src.val[0] = vrev64_u8(_src.val[0]);
- _src.val[1] = vrev64_u8(_src.val[1]);
- _src.val[2] = vrev64_u8(_src.val[2]);
- _src.val[3] = vrev64_u8(_src.val[3]);
-
- _src2.val[0] = vrev64_u8(_src2.val[0]);
- _src2.val[1] = vrev64_u8(_src2.val[1]);
- _src2.val[2] = vrev64_u8(_src2.val[2]);
- _src2.val[3] = vrev64_u8(_src2.val[3]);
-
- vst4_u8(dst0, _src);
- vst4_u8(dst0 - 8 * 4, _src2);
-
- src0 += 16 * 4;
- dst0 -= 16 * 4;
- }
- #else
- if (nn > 0)
- {
- asm volatile(
- "mov r4, #-32 \n"
- "0: \n"
- "pld [%1, #256] \n"
- "vld4.u8 {d0-d3}, [%1]! \n"
- "vrev64.u8 d0, d0 \n"
- "vrev64.u8 d1, d1 \n"
- "vrev64.u8 d2, d2 \n"
- "pld [%1, #256] \n"
- "vld4.u8 {d4-d7}, [%1]! \n"
- "vrev64.u8 d3, d3 \n"
- "vrev64.u8 d4, d4 \n"
- "vrev64.u8 d5, d5 \n"
- "vst4.u8 {d0-d3}, [%2], r4 \n"
- "vrev64.u8 d6, d6 \n"
- "vrev64.u8 d7, d7 \n"
- "subs %0, #1 \n"
- "vst4.u8 {d4-d7}, [%2], r4 \n"
- "bne 0b \n"
- : "=r"(nn), // %0
- "=r"(src0), // %1
- "=r"(dst0) // %2
- : "0"(nn),
- "1"(src0),
- "2"(dst0)
- : "cc", "memory", "q0", "q1", "q2", "q3", "r4");
- }
- #endif // __aarch64__
-
- dst0 += 7 * 4;
- #else
- int remain = srcw;
- #endif // __ARM_NEON
-
- for (; remain > 0; remain--)
- {
- dst0[0] = src0[0];
- dst0[1] = src0[1];
- dst0[2] = src0[2];
- dst0[3] = src0[3];
-
- src0 += 4;
- dst0 -= 4;
- }
-
- src0 += srcwgap;
- dst0 += wgap;
- }
- }
-
- static void kanna_rotate_3_c1(const unsigned char* src, int srcw, int srch, int srcstride, unsigned char* dst, int w, int h, int stride)
- {
- const int srcwgap = srcstride - srcw;
- const int wgap = stride - w;
-
- // point to the last dst pixel
- unsigned char* dstend = dst + stride * h - wgap;
-
- const unsigned char* src0 = src;
- unsigned char* dst0 = dstend - 1;
-
- int y = 0;
- for (; y < srch; y++)
- {
- #if __ARM_NEON
- dst0 -= 15;
-
- int nn = srcw >> 4;
- int remain = srcw - (nn << 4);
-
- #if !NCNN_GNU_INLINE_ASM || __aarch64__
- for (; nn > 0; nn--)
- {
- uint8x8_t _src = vld1_u8(src0);
- uint8x8_t _src2 = vld1_u8(src0 + 8);
-
- _src = vrev64_u8(_src);
- _src2 = vrev64_u8(_src2);
-
- vst1_u8(dst0, _src2);
- vst1_u8(dst0 + 8, _src);
-
- src0 += 16;
- dst0 -= 16;
- }
- #else
- if (nn > 0)
- {
- asm volatile(
- "mov r4, #-16 \n"
- "0: \n"
- "pld [%1, #128] \n"
- "vld1.u8 {d0-d1}, [%1]! \n"
- "vrev64.u8 d3, d0 \n"
- "vrev64.u8 d2, d1 \n"
- "subs %0, #1 \n"
- "vst1.u8 {d2-d3}, [%2], r4 \n"
- "bne 0b \n"
- : "=r"(nn), // %0
- "=r"(src0), // %1
- "=r"(dst0) // %2
- : "0"(nn),
- "1"(src0),
- "2"(dst0)
- : "cc", "memory", "q0", "q1", "r4");
- }
- #endif // __aarch64__
-
- dst0 += 15;
- #else
- int remain = srcw;
- #endif // __ARM_NEON
-
- for (; remain > 0; remain--)
- {
- *dst0 = *src0;
-
- src0 += 1;
- dst0 -= 1;
- }
-
- src0 += srcwgap;
- dst0 -= wgap;
- }
- }
-
- static void kanna_rotate_3_c2(const unsigned char* src, int srcw, int srch, int srcstride, unsigned char* dst, int w, int h, int stride)
- {
- const int srcwgap = srcstride - srcw * 2;
- const int wgap = stride - w * 2;
-
- // point to the last dst pixel
- unsigned char* dstend = dst + stride * h - wgap;
-
- const unsigned char* src0 = src;
- unsigned char* dst0 = dstend - 2;
-
- int y = 0;
- for (; y < srch; y++)
- {
- #if __ARM_NEON
- dst0 -= 7 * 2;
-
- int nn = srcw >> 4;
- int remain = srcw - (nn << 4);
-
- #if !NCNN_GNU_INLINE_ASM || __aarch64__
- for (; nn > 0; nn--)
- {
- uint8x8x2_t _src = vld2_u8(src0);
- uint8x8x2_t _src2 = vld2_u8(src0 + 8 * 2);
-
- _src.val[0] = vrev64_u8(_src.val[0]);
- _src.val[1] = vrev64_u8(_src.val[1]);
-
- _src2.val[0] = vrev64_u8(_src2.val[0]);
- _src2.val[1] = vrev64_u8(_src2.val[1]);
-
- vst2_u8(dst0, _src);
- vst2_u8(dst0 - 8 * 2, _src2);
-
- src0 += 16 * 2;
- dst0 -= 16 * 2;
- }
- #else
- if (nn > 0)
- {
- asm volatile(
- "mov r4, #-16 \n"
- "0: \n"
- "pld [%1, #128] \n"
- "vld2.u8 {d0-d1}, [%1]! \n"
- "vrev64.u8 d0, d0 \n"
- "pld [%1, #128] \n"
- "vld2.u8 {d2-d3}, [%1]! \n"
- "vrev64.u8 d1, d1 \n"
- "vrev64.u8 d2, d2 \n"
- "vst2.u8 {d0-d1}, [%2], r4 \n"
- "vrev64.u8 d3, d3 \n"
- "subs %0, #1 \n"
- "vst2.u8 {d2-d3}, [%2], r4 \n"
- "bne 0b \n"
- : "=r"(nn), // %0
- "=r"(src0), // %1
- "=r"(dst0) // %2
- : "0"(nn),
- "1"(src0),
- "2"(dst0)
- : "cc", "memory", "q0", "q1", "r4");
- }
- #endif // __aarch64__
-
- dst0 += 7 * 2;
- #else
- int remain = srcw;
- #endif // __ARM_NEON
-
- for (; remain > 0; remain--)
- {
- dst0[0] = src0[0];
- dst0[1] = src0[1];
-
- src0 += 2;
- dst0 -= 2;
- }
-
- src0 += srcwgap;
- dst0 -= wgap;
- }
- }
-
- static void kanna_rotate_3_c3(const unsigned char* src, int srcw, int srch, int srcstride, unsigned char* dst, int w, int h, int stride)
- {
- const int srcwgap = srcstride - srcw * 3;
- const int wgap = stride - w * 3;
-
- // point to the last dst pixel
- unsigned char* dstend = dst + stride * h - wgap;
-
- const unsigned char* src0 = src;
- unsigned char* dst0 = dstend - 3;
-
- int y = 0;
- for (; y < srch; y++)
- {
- #if __ARM_NEON
- dst0 -= 7 * 3;
-
- int nn = srcw >> 4;
- int remain = srcw - (nn << 4);
-
- #if !NCNN_GNU_INLINE_ASM || __aarch64__
- for (; nn > 0; nn--)
- {
- uint8x8x3_t _src = vld3_u8(src0);
- uint8x8x3_t _src2 = vld3_u8(src0 + 8 * 3);
-
- _src.val[0] = vrev64_u8(_src.val[0]);
- _src.val[1] = vrev64_u8(_src.val[1]);
- _src.val[2] = vrev64_u8(_src.val[2]);
-
- _src2.val[0] = vrev64_u8(_src2.val[0]);
- _src2.val[1] = vrev64_u8(_src2.val[1]);
- _src2.val[2] = vrev64_u8(_src2.val[2]);
-
- vst3_u8(dst0, _src);
- vst3_u8(dst0 - 8 * 3, _src2);
-
- src0 += 16 * 3;
- dst0 -= 16 * 3;
- }
- #else
- if (nn > 0)
- {
- asm volatile(
- "mov r4, #-24 \n"
- "0: \n"
- "pld [%1, #192] \n"
- "vld3.u8 {d0-d2}, [%1]! \n"
- "vrev64.u8 d0, d0 \n"
- "vrev64.u8 d1, d1 \n"
- "pld [%1, #192] \n"
- "vld3.u8 {d4-d6}, [%1]! \n"
- "vrev64.u8 d2, d2 \n"
- "vrev64.u8 d4, d4 \n"
- "vst3.u8 {d0-d2}, [%2], r4 \n"
- "vrev64.u8 d5, d5 \n"
- "vrev64.u8 d6, d6 \n"
- "subs %0, #1 \n"
- "vst3.u8 {d4-d6}, [%2], r4 \n"
- "bne 0b \n"
- : "=r"(nn), // %0
- "=r"(src0), // %1
- "=r"(dst0) // %2
- : "0"(nn),
- "1"(src0),
- "2"(dst0)
- : "cc", "memory", "q0", "q1", "q2", "q3", "r4");
- }
- #endif // __aarch64__
-
- dst0 += 7 * 3;
- #else
- int remain = srcw;
- #endif // __ARM_NEON
-
- for (; remain > 0; remain--)
- {
- dst0[0] = src0[0];
- dst0[1] = src0[1];
- dst0[2] = src0[2];
-
- src0 += 3;
- dst0 -= 3;
- }
-
- src0 += srcwgap;
- dst0 -= wgap;
- }
- }
-
- static void kanna_rotate_3_c4(const unsigned char* src, int srcw, int srch, int srcstride, unsigned char* dst, int w, int h, int stride)
- {
- const int srcwgap = srcstride - srcw * 4;
- const int wgap = stride - w * 4;
-
- // point to the last dst pixel
- unsigned char* dstend = dst + stride * h - wgap;
-
- const unsigned char* src0 = src;
- unsigned char* dst0 = dstend - 4;
-
- int y = 0;
- for (; y < srch; y++)
- {
- #if __ARM_NEON
- dst0 -= 7 * 4;
-
- int nn = srcw >> 4;
- int remain = srcw - (nn << 4);
-
- #if !NCNN_GNU_INLINE_ASM || __aarch64__
- for (; nn > 0; nn--)
- {
- uint8x8x4_t _src = vld4_u8(src0);
- uint8x8x4_t _src2 = vld4_u8(src0 + 8 * 4);
-
- _src.val[0] = vrev64_u8(_src.val[0]);
- _src.val[1] = vrev64_u8(_src.val[1]);
- _src.val[2] = vrev64_u8(_src.val[2]);
- _src.val[3] = vrev64_u8(_src.val[3]);
-
- _src2.val[0] = vrev64_u8(_src2.val[0]);
- _src2.val[1] = vrev64_u8(_src2.val[1]);
- _src2.val[2] = vrev64_u8(_src2.val[2]);
- _src2.val[3] = vrev64_u8(_src2.val[3]);
-
- vst4_u8(dst0, _src);
- vst4_u8(dst0 - 8 * 4, _src2);
-
- src0 += 16 * 4;
- dst0 -= 16 * 4;
- }
- #else
- if (nn > 0)
- {
- asm volatile(
- "mov r4, #-32 \n"
- "0: \n"
- "pld [%1, #256] \n"
- "vld4.u8 {d0-d3}, [%1]! \n"
- "vrev64.u8 d0, d0 \n"
- "vrev64.u8 d1, d1 \n"
- "vrev64.u8 d2, d2 \n"
- "pld [%1, #256] \n"
- "vld4.u8 {d4-d7}, [%1]! \n"
- "vrev64.u8 d3, d3 \n"
- "vrev64.u8 d4, d4 \n"
- "vrev64.u8 d5, d5 \n"
- "vst4.u8 {d0-d3}, [%2], r4 \n"
- "vrev64.u8 d6, d6 \n"
- "vrev64.u8 d7, d7 \n"
- "subs %0, #1 \n"
- "vst4.u8 {d4-d7}, [%2], r4 \n"
- "bne 0b \n"
- : "=r"(nn), // %0
- "=r"(src0), // %1
- "=r"(dst0) // %2
- : "0"(nn),
- "1"(src0),
- "2"(dst0)
- : "cc", "memory", "q0", "q1", "q2", "q3", "r4");
- }
- #endif // __aarch64__
-
- dst0 += 7 * 4;
- #else
- int remain = srcw;
- #endif // __ARM_NEON
-
- for (; remain > 0; remain--)
- {
- dst0[0] = src0[0];
- dst0[1] = src0[1];
- dst0[2] = src0[2];
- dst0[3] = src0[3];
-
- src0 += 4;
- dst0 -= 4;
- }
-
- src0 += srcwgap;
- dst0 -= wgap;
- }
- }
-
- static void kanna_rotate_4_c1(const unsigned char* src, int srcw, int srch, int srcstride, unsigned char* dst, int w, int h, int stride)
- {
- const int srcwgap = srcstride - srcw;
- const int wgap = stride + w;
-
- // point to the last dst pixel row
- unsigned char* dstend = dst + stride * (h - 1);
-
- const unsigned char* src0 = src;
- const unsigned char* src1 = src + srcstride;
- unsigned char* dst0 = dstend;
- unsigned char* dst1 = dstend - stride;
-
- int y = 0;
- for (; y + 1 < srch; y += 2)
- {
- #if __ARM_NEON
- int nn = srcw >> 5;
- int remain = srcw - (nn << 5);
- #if !NCNN_GNU_INLINE_ASM || __aarch64__
- for (; nn > 0; nn--)
- {
- uint8x16_t _src0 = vld1q_u8(src0);
- uint8x16_t _src0n = vld1q_u8(src0 + 16);
- vst1q_u8(dst0, _src0);
- vst1q_u8(dst0 + 16, _src0n);
-
- uint8x16_t _src1 = vld1q_u8(src1);
- uint8x16_t _src1n = vld1q_u8(src1 + 16);
- vst1q_u8(dst1, _src1);
- vst1q_u8(dst1 + 16, _src1n);
-
- src0 += 32;
- src1 += 32;
- dst0 += 32;
- dst1 += 32;
- }
- #else
- if (nn > 0)
- {
- asm volatile(
- "0: \n"
- "pld [%1, #256] \n"
- "vld1.u8 {d0-d3}, [%1]! \n"
- "pld [%2, #256] \n"
- "vld1.u8 {d4-d7}, [%2]! \n"
- "subs %0, #1 \n"
- "vst1.u8 {d0-d3}, [%3]! \n"
- "vst1.u8 {d4-d7}, [%4]! \n"
- "bne 0b \n"
- : "=r"(nn), // %0
- "=r"(src0), // %1
- "=r"(src1), // %2
- "=r"(dst0), // %3
- "=r"(dst1) // %4
- : "0"(nn),
- "1"(src0),
- "2"(src1),
- "3"(dst0),
- "4"(dst1)
- : "cc", "memory", "q0", "q1", "q2", "q3");
- }
- #endif // __aarch64__
- #else
- int remain = srcw;
- #endif // __ARM_NEON
-
- for (; remain > 0; remain--)
- {
- *dst0++ = *src0++;
- *dst1++ = *src1++;
- }
-
- src0 += srcwgap + srcstride;
- src1 += srcwgap + srcstride;
- dst0 -= wgap + stride;
- dst1 -= wgap + stride;
- }
-
- for (; y < srch; y++)
- {
- #if __ARM_NEON
- int nn = srcw >> 5;
- int remain = srcw - (nn << 5);
- #if !NCNN_GNU_INLINE_ASM || __aarch64__
- for (; nn > 0; nn--)
- {
- uint8x16_t _src = vld1q_u8(src0);
- uint8x16_t _src2 = vld1q_u8(src0 + 16);
- vst1q_u8(dst0, _src);
- vst1q_u8(dst0 + 16, _src2);
-
- src0 += 32;
- dst0 += 32;
- }
- #else
- if (nn > 0)
- {
- asm volatile(
- "0: \n"
- "pld [%1, #256] \n"
- "vld1.u8 {d0-d3}, [%1]! \n"
- "subs %0, #1 \n"
- "vst1.u8 {d0-d3}, [%2]! \n"
- "bne 0b \n"
- : "=r"(nn), // %0
- "=r"(src0), // %1
- "=r"(dst0) // %2
- : "0"(nn),
- "1"(src0),
- "2"(dst0)
- : "cc", "memory", "q0", "q1");
- }
- #endif // __aarch64__
- #else
- int remain = srcw;
- #endif // __ARM_NEON
-
- for (; remain > 0; remain--)
- {
- *dst0++ = *src0++;
- }
-
- src0 += srcwgap;
- dst0 -= wgap;
- }
- }
-
- static void kanna_rotate_4_c2(const unsigned char* src, int srcw, int srch, int srcstride, unsigned char* dst, int w, int h, int stride)
- {
- const int srcwgap = srcstride - srcw * 2;
- const int wgap = stride + w * 2;
-
- // point to the last dst pixel row
- unsigned char* dstend = dst + stride * (h - 1);
-
- int size = srcw * 2;
-
- const unsigned char* src0 = src;
- const unsigned char* src1 = src + srcstride;
- unsigned char* dst0 = dstend;
- unsigned char* dst1 = dstend - stride;
-
- int y = 0;
- for (; y + 1 < srch; y += 2)
- {
- #if __ARM_NEON
- int nn = size >> 5;
- int remain = size - (nn << 5);
- #if !NCNN_GNU_INLINE_ASM || __aarch64__
- for (; nn > 0; nn--)
- {
- uint8x16_t _src0 = vld1q_u8(src0);
- uint8x16_t _src0n = vld1q_u8(src0 + 16);
- vst1q_u8(dst0, _src0);
- vst1q_u8(dst0 + 16, _src0n);
-
- uint8x16_t _src1 = vld1q_u8(src1);
- uint8x16_t _src1n = vld1q_u8(src1 + 16);
- vst1q_u8(dst1, _src1);
- vst1q_u8(dst1 + 16, _src1n);
-
- src0 += 32;
- src1 += 32;
- dst0 += 32;
- dst1 += 32;
- }
- #else
- if (nn > 0)
- {
- asm volatile(
- "0: \n"
- "pld [%1, #256] \n"
- "vld1.u8 {d0-d3}, [%1]! \n"
- "pld [%2, #256] \n"
- "vld1.u8 {d4-d7}, [%2]! \n"
- "subs %0, #1 \n"
- "vst1.u8 {d0-d3}, [%3]! \n"
- "vst1.u8 {d4-d7}, [%4]! \n"
- "bne 0b \n"
- : "=r"(nn), // %0
- "=r"(src0), // %1
- "=r"(src1), // %2
- "=r"(dst0), // %3
- "=r"(dst1) // %4
- : "0"(nn),
- "1"(src0),
- "2"(src1),
- "3"(dst0),
- "4"(dst1)
- : "cc", "memory", "q0", "q1", "q2", "q3");
- }
- #endif // __aarch64__
- #else
- int remain = size;
- #endif // __ARM_NEON
-
- for (; remain > 0; remain--)
- {
- *dst0++ = *src0++;
- *dst1++ = *src1++;
- }
-
- src0 += srcwgap + srcstride;
- src1 += srcwgap + srcstride;
- dst0 -= wgap + stride;
- dst1 -= wgap + stride;
- }
-
- for (; y < srch; y++)
- {
- #if __ARM_NEON
- int nn = size >> 5;
- int remain = size - (nn << 5);
- #if !NCNN_GNU_INLINE_ASM || __aarch64__
- for (; nn > 0; nn--)
- {
- uint8x16_t _src = vld1q_u8(src0);
- uint8x16_t _src2 = vld1q_u8(src0 + 16);
- vst1q_u8(dst0, _src);
- vst1q_u8(dst0 + 16, _src2);
-
- src0 += 32;
- dst0 += 32;
- }
- #else
- if (nn > 0)
- {
- asm volatile(
- "0: \n"
- "pld [%1, #256] \n"
- "vld1.u8 {d0-d3}, [%1]! \n"
- "subs %0, #1 \n"
- "vst1.u8 {d0-d3}, [%2]! \n"
- "bne 0b \n"
- : "=r"(nn), // %0
- "=r"(src0), // %1
- "=r"(dst0) // %2
- : "0"(nn),
- "1"(src0),
- "2"(dst0)
- : "cc", "memory", "q0", "q1");
- }
- #endif // __aarch64__
- #else
- int remain = size;
- #endif // __ARM_NEON
-
- for (; remain > 0; remain--)
- {
- *dst0++ = *src0++;
- }
-
- src0 += srcwgap;
- dst0 -= wgap;
- }
- }
-
- static void kanna_rotate_4_c3(const unsigned char* src, int srcw, int srch, int srcstride, unsigned char* dst, int w, int h, int stride)
- {
- const int srcwgap = srcstride - srcw * 3;
- const int wgap = stride + w * 3;
-
- // point to the last dst pixel row
- unsigned char* dstend = dst + stride * (h - 1);
-
- int size = srcw * 3;
-
- const unsigned char* src0 = src;
- const unsigned char* src1 = src + srcstride;
- unsigned char* dst0 = dstend;
- unsigned char* dst1 = dstend - stride;
-
- int y = 0;
- for (; y + 1 < srch; y += 2)
- {
- #if __ARM_NEON
- int nn = size >> 5;
- int remain = size - (nn << 5);
- #if !NCNN_GNU_INLINE_ASM || __aarch64__
- for (; nn > 0; nn--)
- {
- uint8x16_t _src0 = vld1q_u8(src0);
- uint8x16_t _src0n = vld1q_u8(src0 + 16);
- vst1q_u8(dst0, _src0);
- vst1q_u8(dst0 + 16, _src0n);
-
- uint8x16_t _src1 = vld1q_u8(src1);
- uint8x16_t _src1n = vld1q_u8(src1 + 16);
- vst1q_u8(dst1, _src1);
- vst1q_u8(dst1 + 16, _src1n);
-
- src0 += 32;
- src1 += 32;
- dst0 += 32;
- dst1 += 32;
- }
- #else
- if (nn > 0)
- {
- asm volatile(
- "0: \n"
- "pld [%1, #256] \n"
- "vld1.u8 {d0-d3}, [%1]! \n"
- "pld [%2, #256] \n"
- "vld1.u8 {d4-d7}, [%2]! \n"
- "subs %0, #1 \n"
- "vst1.u8 {d0-d3}, [%3]! \n"
- "vst1.u8 {d4-d7}, [%4]! \n"
- "bne 0b \n"
- : "=r"(nn), // %0
- "=r"(src0), // %1
- "=r"(src1), // %2
- "=r"(dst0), // %3
- "=r"(dst1) // %4
- : "0"(nn),
- "1"(src0),
- "2"(src1),
- "3"(dst0),
- "4"(dst1)
- : "cc", "memory", "q0", "q1", "q2", "q3");
- }
- #endif // __aarch64__
- #else
- int remain = size;
- #endif // __ARM_NEON
-
- for (; remain > 0; remain--)
- {
- *dst0++ = *src0++;
- *dst1++ = *src1++;
- }
-
- src0 += srcwgap + srcstride;
- src1 += srcwgap + srcstride;
- dst0 -= wgap + stride;
- dst1 -= wgap + stride;
- }
-
- for (; y < srch; y++)
- {
- #if __ARM_NEON
- int nn = size >> 5;
- int remain = size - (nn << 5);
- #if !NCNN_GNU_INLINE_ASM || __aarch64__
- for (; nn > 0; nn--)
- {
- uint8x16_t _src = vld1q_u8(src0);
- uint8x16_t _src2 = vld1q_u8(src0 + 16);
- vst1q_u8(dst0, _src);
- vst1q_u8(dst0 + 16, _src2);
-
- src0 += 32;
- dst0 += 32;
- }
- #else
- if (nn > 0)
- {
- asm volatile(
- "0: \n"
- "pld [%1, #256] \n"
- "vld1.u8 {d0-d3}, [%1]! \n"
- "subs %0, #1 \n"
- "vst1.u8 {d0-d3}, [%2]! \n"
- "bne 0b \n"
- : "=r"(nn), // %0
- "=r"(src0), // %1
- "=r"(dst0) // %2
- : "0"(nn),
- "1"(src0),
- "2"(dst0)
- : "cc", "memory", "q0", "q1");
- }
- #endif // __aarch64__
- #else
- int remain = size;
- #endif // __ARM_NEON
-
- for (; remain > 0; remain--)
- {
- *dst0++ = *src0++;
- }
-
- src0 += srcwgap;
- dst0 -= wgap;
- }
- }
-
- static void kanna_rotate_4_c4(const unsigned char* src, int srcw, int srch, int srcstride, unsigned char* dst, int w, int h, int stride)
- {
- const int srcwgap = srcstride - srcw * 4;
- const int wgap = stride + w * 4;
-
- // point to the last dst pixel row
- unsigned char* dstend = dst + stride * (h - 1);
-
- int size = srcw * 4;
-
- const unsigned char* src0 = src;
- const unsigned char* src1 = src + srcstride;
- unsigned char* dst0 = dstend;
- unsigned char* dst1 = dstend - stride;
-
- int y = 0;
- for (; y + 1 < srch; y += 2)
- {
- #if __ARM_NEON
- int nn = size >> 5;
- int remain = size - (nn << 5);
- #if !NCNN_GNU_INLINE_ASM || __aarch64__
- for (; nn > 0; nn--)
- {
- uint8x16_t _src0 = vld1q_u8(src0);
- uint8x16_t _src0n = vld1q_u8(src0 + 16);
- vst1q_u8(dst0, _src0);
- vst1q_u8(dst0 + 16, _src0n);
-
- uint8x16_t _src1 = vld1q_u8(src1);
- uint8x16_t _src1n = vld1q_u8(src1 + 16);
- vst1q_u8(dst1, _src1);
- vst1q_u8(dst1 + 16, _src1n);
-
- src0 += 32;
- src1 += 32;
- dst0 += 32;
- dst1 += 32;
- }
- #else
- if (nn > 0)
- {
- asm volatile(
- "0: \n"
- "pld [%1, #256] \n"
- "vld1.u8 {d0-d3}, [%1]! \n"
- "pld [%2, #256] \n"
- "vld1.u8 {d4-d7}, [%2]! \n"
- "subs %0, #1 \n"
- "vst1.u8 {d0-d3}, [%3]! \n"
- "vst1.u8 {d4-d7}, [%4]! \n"
- "bne 0b \n"
- : "=r"(nn), // %0
- "=r"(src0), // %1
- "=r"(src1), // %2
- "=r"(dst0), // %3
- "=r"(dst1) // %4
- : "0"(nn),
- "1"(src0),
- "2"(src1),
- "3"(dst0),
- "4"(dst1)
- : "cc", "memory", "q0", "q1", "q2", "q3");
- }
- #endif // __aarch64__
- #else
- int remain = size;
- #endif // __ARM_NEON
-
- for (; remain > 0; remain--)
- {
- *dst0++ = *src0++;
- *dst1++ = *src1++;
- }
-
- src0 += srcwgap + srcstride;
- src1 += srcwgap + srcstride;
- dst0 -= wgap + stride;
- dst1 -= wgap + stride;
- }
-
- for (; y < srch; y++)
- {
- #if __ARM_NEON
- int nn = size >> 5;
- int remain = size - (nn << 5);
- #if !NCNN_GNU_INLINE_ASM || __aarch64__
- for (; nn > 0; nn--)
- {
- uint8x16_t _src = vld1q_u8(src0);
- uint8x16_t _src2 = vld1q_u8(src0 + 16);
- vst1q_u8(dst0, _src);
- vst1q_u8(dst0 + 16, _src2);
-
- src0 += 32;
- dst0 += 32;
- }
- #else
- if (nn > 0)
- {
- asm volatile(
- "0: \n"
- "pld [%1, #256] \n"
- "vld1.u8 {d0-d3}, [%1]! \n"
- "subs %0, #1 \n"
- "vst1.u8 {d0-d3}, [%2]! \n"
- "bne 0b \n"
- : "=r"(nn), // %0
- "=r"(src0), // %1
- "=r"(dst0) // %2
- : "0"(nn),
- "1"(src0),
- "2"(dst0)
- : "cc", "memory", "q0", "q1");
- }
- #endif // __aarch64__
- #else
- int remain = size;
- #endif // __ARM_NEON
-
- for (; remain > 0; remain--)
- {
- *dst0++ = *src0++;
- }
-
- src0 += srcwgap;
- dst0 -= wgap;
- }
- }
-
- static void kanna_rotate_5_c1(const unsigned char* src, int srcw, int srch, int srcstride, unsigned char* dst, int /*w*/, int /*h*/, int stride)
- {
- const int srcwgap = srcstride - srcw;
-
- const unsigned char* src0 = src;
-
- int y = 0;
- #if __ARM_NEON
- for (; y + 7 < srch; y += 8)
- {
- const unsigned char* src1 = src0 + srcstride;
-
- unsigned char* dst0 = dst + y;
- unsigned char* dst1 = dst + y + stride;
-
- int src_step = 2 * srcstride;
- int dst_step = 2 * stride;
-
- int nn = srcw >> 3;
- int remain = srcw - (nn << 3);
-
- #if !NCNN_GNU_INLINE_ASM || __aarch64__
- for (; nn > 0; nn--)
- {
- uint8x8_t _src0 = vld1_u8(src0);
- uint8x8_t _src1 = vld1_u8(src1);
-
- uint8x8_t _src2 = vld1_u8(src0 + src_step);
- uint8x8_t _src3 = vld1_u8(src1 + src_step);
-
- uint8x8_t _src4 = vld1_u8(src0 + 2 * src_step);
- uint8x8_t _src5 = vld1_u8(src1 + 2 * src_step);
-
- uint8x8_t _src6 = vld1_u8(src0 + 3 * src_step);
- uint8x8_t _src7 = vld1_u8(src1 + 3 * src_step);
-
- uint8x8x2_t _src01t_r = vtrn_u8(_src0, _src1);
- uint8x8x2_t _src23t_r = vtrn_u8(_src2, _src3);
- uint8x8x2_t _src45t_r = vtrn_u8(_src4, _src5);
- uint8x8x2_t _src67t_r = vtrn_u8(_src6, _src7);
-
- uint16x4x2_t _src02tt_r = vtrn_u16(vreinterpret_u16_u8(_src01t_r.val[0]), vreinterpret_u16_u8(_src23t_r.val[0]));
- uint16x4x2_t _src13tt_r = vtrn_u16(vreinterpret_u16_u8(_src01t_r.val[1]), vreinterpret_u16_u8(_src23t_r.val[1]));
- uint16x4x2_t _src46tt_r = vtrn_u16(vreinterpret_u16_u8(_src45t_r.val[0]), vreinterpret_u16_u8(_src67t_r.val[0]));
- uint16x4x2_t _src57tt_r = vtrn_u16(vreinterpret_u16_u8(_src45t_r.val[1]), vreinterpret_u16_u8(_src67t_r.val[1]));
-
- uint32x2x2_t _src04ttt_r = vtrn_u32(vreinterpret_u32_u16(_src02tt_r.val[0]), vreinterpret_u32_u16(_src46tt_r.val[0]));
- uint32x2x2_t _src15ttt_r = vtrn_u32(vreinterpret_u32_u16(_src13tt_r.val[0]), vreinterpret_u32_u16(_src57tt_r.val[0]));
- uint32x2x2_t _src26ttt_r = vtrn_u32(vreinterpret_u32_u16(_src02tt_r.val[1]), vreinterpret_u32_u16(_src46tt_r.val[1]));
- uint32x2x2_t _src37ttt_r = vtrn_u32(vreinterpret_u32_u16(_src13tt_r.val[1]), vreinterpret_u32_u16(_src57tt_r.val[1]));
-
- uint8x8_t _dst0 = vreinterpret_u8_u32(_src04ttt_r.val[0]);
- uint8x8_t _dst1 = vreinterpret_u8_u32(_src15ttt_r.val[0]);
- uint8x8_t _dst2 = vreinterpret_u8_u32(_src26ttt_r.val[0]);
- uint8x8_t _dst3 = vreinterpret_u8_u32(_src37ttt_r.val[0]);
- uint8x8_t _dst4 = vreinterpret_u8_u32(_src04ttt_r.val[1]);
- uint8x8_t _dst5 = vreinterpret_u8_u32(_src15ttt_r.val[1]);
- uint8x8_t _dst6 = vreinterpret_u8_u32(_src26ttt_r.val[1]);
- uint8x8_t _dst7 = vreinterpret_u8_u32(_src37ttt_r.val[1]);
-
- vst1_u8(dst0, _dst0);
- vst1_u8(dst1, _dst1);
- vst1_u8(dst0 + dst_step, _dst2);
- vst1_u8(dst1 + dst_step, _dst3);
- vst1_u8(dst0 + 2 * dst_step, _dst4);
- vst1_u8(dst1 + 2 * dst_step, _dst5);
- vst1_u8(dst0 + 3 * dst_step, _dst6);
- vst1_u8(dst1 + 3 * dst_step, _dst7);
-
- src0 += 8;
- src1 += 8;
-
- dst0 += 4 * dst_step;
- dst1 += 4 * dst_step;
- }
- #else
- if (nn > 0)
- {
- asm volatile(
- "0: \n"
- "pld [%1, #64] \n"
- "vld1.u8 {d0}, [%1], %10 \n"
-
- "pld [%2, #64] \n"
- "vld1.u8 {d1}, [%2], %10 \n"
-
- "pld [%1, #64] \n"
- "vld1.u8 {d2}, [%1], %10 \n"
-
- "vtrn.u8 d0, d1 \n" // _src01t_r
-
- "pld [%2, #64] \n"
- "vld1.u8 {d3}, [%2], %10 \n"
-
- "pld [%1, #64] \n"
- "vld1.u8 {d4}, [%1], %10 \n"
-
- "vtrn.u8 d2, d3 \n" // _src23t_r
-
- "pld [%2, #64] \n"
- "vld1.u8 {d5}, [%2], %10 \n"
-
- "pld [%1, #64] \n"
- "vld1.u8 {d6}, [%1], %10 \n"
-
- "vtrn.u8 d4, d5 \n" // _src45t_r
-
- "pld [%2, #64] \n"
- "vld1.u8 {d7}, [%2], %10 \n"
-
- "vtrn.u8 d6, d7 \n" // _src67t_r
-
- "sub %1, %1, %10, lsl #2 \n" // restore src0
-
- "vtrn.u16 q0, q1 \n" // _src02tt_r _src13tt_r
-
- "sub %2, %2, %10, lsl #2 \n" // restore src1
-
- "vtrn.u16 q2, q3 \n" // _src13tt_r _src46tt_r
-
- "add %1, #8 \n" // src0 += 8
-
- "vtrn.u32 q0, q2 \n" // _src04ttt_r _src15ttt_r
-
- "add %2, #8 \n" // src1 += 8
-
- "vtrn.u32 q1, q3 \n" // _src26ttt_r _src37ttt_r
- "vst1.u8 {d0}, [%3], %11 \n"
- "vst1.u8 {d1}, [%4], %11 \n"
-
- "subs %0, #1 \n"
-
- "vst1.u8 {d2}, [%3], %11 \n"
- "vst1.u8 {d3}, [%4], %11 \n"
- "vst1.u8 {d4}, [%3], %11 \n"
- "vst1.u8 {d5}, [%4], %11 \n"
- "vst1.u8 {d6}, [%3], %11 \n"
- "vst1.u8 {d7}, [%4], %11 \n"
-
- "bne 0b \n"
- : "=r"(nn), // %0
- "=r"(src0), // %1
- "=r"(src1), // %2
- "=r"(dst0), // %3
- "=r"(dst1) // %4
- : "0"(nn),
- "1"(src0),
- "2"(src1),
- "3"(dst0),
- "4"(dst1),
- "r"(src_step), // %10
- "r"(dst_step) // %11
- : "cc", "memory", "q0", "q1", "q2", "q3");
- }
- #endif // __aarch64__
- for (; remain > 0; remain--)
- {
- dst0[0] = src0[0];
- dst0[1] = src1[0];
- dst0[2] = src0[0 + src_step];
- dst0[3] = src1[0 + src_step];
- dst0[4] = src0[0 + 2 * src_step];
- dst0[5] = src1[0 + 2 * src_step];
- dst0[6] = src0[0 + 3 * src_step];
- dst0[7] = src1[0 + 3 * src_step];
-
- src0 += 1;
- src1 += 1;
-
- dst0 += stride;
- }
-
- src0 += srcwgap + 7 * srcstride;
- }
- #endif // __ARM_NEON
- for (; y < srch; y++)
- {
- unsigned char* dst0 = dst + y;
-
- int x = 0;
- for (; x < srcw; x++)
- {
- *dst0 = *src0;
-
- src0 += 1;
- dst0 += stride;
- }
-
- src0 += srcwgap;
- }
- }
-
- static void kanna_rotate_5_c2(const unsigned char* src, int srcw, int srch, int srcstride, unsigned char* dst, int /*w*/, int /*h*/, int stride)
- {
- const int srcwgap = srcstride - srcw * 2;
-
- const unsigned char* src0 = src;
-
- int y = 0;
- #if __ARM_NEON
- for (; y + 7 < srch; y += 8)
- {
- const unsigned char* src1 = src0 + srcstride;
-
- unsigned char* dst0 = dst + y * 2;
- unsigned char* dst1 = dst + y * 2 + stride;
-
- int src_step = 2 * srcstride;
- int dst_step = 2 * stride;
-
- int nn = srcw >> 3;
- int remain = srcw - (nn << 3);
-
- #if !NCNN_GNU_INLINE_ASM || __aarch64__
- for (; nn > 0; nn--)
- {
- uint8x8x2_t _src0 = vld2_u8(src0);
- uint8x8x2_t _src1 = vld2_u8(src1);
-
- uint8x8x2_t _src2 = vld2_u8(src0 + src_step);
- uint8x8x2_t _src3 = vld2_u8(src1 + src_step);
-
- uint8x8x2_t _src4 = vld2_u8(src0 + 2 * src_step);
- uint8x8x2_t _src5 = vld2_u8(src1 + 2 * src_step);
-
- uint8x8x2_t _src6 = vld2_u8(src0 + 3 * src_step);
- uint8x8x2_t _src7 = vld2_u8(src1 + 3 * src_step);
-
- uint8x8x2_t _src01t_r = vtrn_u8(_src0.val[0], _src1.val[0]);
- uint8x8x2_t _src23t_r = vtrn_u8(_src2.val[0], _src3.val[0]);
- uint8x8x2_t _src45t_r = vtrn_u8(_src4.val[0], _src5.val[0]);
- uint8x8x2_t _src67t_r = vtrn_u8(_src6.val[0], _src7.val[0]);
-
- uint8x8x2_t _src01t_g = vtrn_u8(_src0.val[1], _src1.val[1]);
- uint8x8x2_t _src23t_g = vtrn_u8(_src2.val[1], _src3.val[1]);
- uint8x8x2_t _src45t_g = vtrn_u8(_src4.val[1], _src5.val[1]);
- uint8x8x2_t _src67t_g = vtrn_u8(_src6.val[1], _src7.val[1]);
-
- uint16x4x2_t _src02tt_r = vtrn_u16(vreinterpret_u16_u8(_src01t_r.val[0]), vreinterpret_u16_u8(_src23t_r.val[0]));
- uint16x4x2_t _src13tt_r = vtrn_u16(vreinterpret_u16_u8(_src01t_r.val[1]), vreinterpret_u16_u8(_src23t_r.val[1]));
- uint16x4x2_t _src46tt_r = vtrn_u16(vreinterpret_u16_u8(_src45t_r.val[0]), vreinterpret_u16_u8(_src67t_r.val[0]));
- uint16x4x2_t _src57tt_r = vtrn_u16(vreinterpret_u16_u8(_src45t_r.val[1]), vreinterpret_u16_u8(_src67t_r.val[1]));
-
- uint16x4x2_t _src02tt_g = vtrn_u16(vreinterpret_u16_u8(_src01t_g.val[0]), vreinterpret_u16_u8(_src23t_g.val[0]));
- uint16x4x2_t _src13tt_g = vtrn_u16(vreinterpret_u16_u8(_src01t_g.val[1]), vreinterpret_u16_u8(_src23t_g.val[1]));
- uint16x4x2_t _src46tt_g = vtrn_u16(vreinterpret_u16_u8(_src45t_g.val[0]), vreinterpret_u16_u8(_src67t_g.val[0]));
- uint16x4x2_t _src57tt_g = vtrn_u16(vreinterpret_u16_u8(_src45t_g.val[1]), vreinterpret_u16_u8(_src67t_g.val[1]));
-
- uint32x2x2_t _src04ttt_r = vtrn_u32(vreinterpret_u32_u16(_src02tt_r.val[0]), vreinterpret_u32_u16(_src46tt_r.val[0]));
- uint32x2x2_t _src15ttt_r = vtrn_u32(vreinterpret_u32_u16(_src13tt_r.val[0]), vreinterpret_u32_u16(_src57tt_r.val[0]));
- uint32x2x2_t _src26ttt_r = vtrn_u32(vreinterpret_u32_u16(_src02tt_r.val[1]), vreinterpret_u32_u16(_src46tt_r.val[1]));
- uint32x2x2_t _src37ttt_r = vtrn_u32(vreinterpret_u32_u16(_src13tt_r.val[1]), vreinterpret_u32_u16(_src57tt_r.val[1]));
-
- uint32x2x2_t _src04ttt_g = vtrn_u32(vreinterpret_u32_u16(_src02tt_g.val[0]), vreinterpret_u32_u16(_src46tt_g.val[0]));
- uint32x2x2_t _src15ttt_g = vtrn_u32(vreinterpret_u32_u16(_src13tt_g.val[0]), vreinterpret_u32_u16(_src57tt_g.val[0]));
- uint32x2x2_t _src26ttt_g = vtrn_u32(vreinterpret_u32_u16(_src02tt_g.val[1]), vreinterpret_u32_u16(_src46tt_g.val[1]));
- uint32x2x2_t _src37ttt_g = vtrn_u32(vreinterpret_u32_u16(_src13tt_g.val[1]), vreinterpret_u32_u16(_src57tt_g.val[1]));
-
- uint8x8x2_t _dst0;
- uint8x8x2_t _dst1;
- uint8x8x2_t _dst2;
- uint8x8x2_t _dst3;
- uint8x8x2_t _dst4;
- uint8x8x2_t _dst5;
- uint8x8x2_t _dst6;
- uint8x8x2_t _dst7;
-
- _dst0.val[0] = vreinterpret_u8_u32(_src04ttt_r.val[0]);
- _dst1.val[0] = vreinterpret_u8_u32(_src15ttt_r.val[0]);
- _dst2.val[0] = vreinterpret_u8_u32(_src26ttt_r.val[0]);
- _dst3.val[0] = vreinterpret_u8_u32(_src37ttt_r.val[0]);
- _dst4.val[0] = vreinterpret_u8_u32(_src04ttt_r.val[1]);
- _dst5.val[0] = vreinterpret_u8_u32(_src15ttt_r.val[1]);
- _dst6.val[0] = vreinterpret_u8_u32(_src26ttt_r.val[1]);
- _dst7.val[0] = vreinterpret_u8_u32(_src37ttt_r.val[1]);
-
- _dst0.val[1] = vreinterpret_u8_u32(_src04ttt_g.val[0]);
- _dst1.val[1] = vreinterpret_u8_u32(_src15ttt_g.val[0]);
- _dst2.val[1] = vreinterpret_u8_u32(_src26ttt_g.val[0]);
- _dst3.val[1] = vreinterpret_u8_u32(_src37ttt_g.val[0]);
- _dst4.val[1] = vreinterpret_u8_u32(_src04ttt_g.val[1]);
- _dst5.val[1] = vreinterpret_u8_u32(_src15ttt_g.val[1]);
- _dst6.val[1] = vreinterpret_u8_u32(_src26ttt_g.val[1]);
- _dst7.val[1] = vreinterpret_u8_u32(_src37ttt_g.val[1]);
-
- vst2_u8(dst0, _dst0);
- vst2_u8(dst1, _dst1);
- vst2_u8(dst0 + dst_step, _dst2);
- vst2_u8(dst1 + dst_step, _dst3);
- vst2_u8(dst0 + 2 * dst_step, _dst4);
- vst2_u8(dst1 + 2 * dst_step, _dst5);
- vst2_u8(dst0 + 3 * dst_step, _dst6);
- vst2_u8(dst1 + 3 * dst_step, _dst7);
-
- src0 += 2 * 8;
- src1 += 2 * 8;
-
- dst0 += 4 * dst_step;
- dst1 += 4 * dst_step;
- }
- #else
- if (nn > 0)
- {
- asm volatile(
- "0: \n"
- "pld [%1, #128] \n"
- "vld2.u8 {d0-d1}, [%1], %10 \n"
-
- "pld [%2, #128] \n"
- "vld2.u8 {d2-d3}, [%2], %10 \n"
-
- "pld [%1, #128] \n"
- "vld2.u8 {d4-d5}, [%1], %10 \n"
-
- "vtrn.u8 q0, q1 \n" // _src01t_r
-
- "pld [%2, #128] \n"
- "vld2.u8 {d6-d7}, [%2], %10 \n"
-
- "pld [%1, #128] \n"
- "vld2.u8 {d16-d17}, [%1], %10\n"
-
- "vtrn.u8 q2, q3 \n" // _src23t_r
-
- "pld [%2, #128] \n"
- "vld2.u8 {d18-d19}, [%2], %10\n"
-
- "pld [%1, #128] \n"
- "vld2.u8 {d20-d21}, [%1], %10\n"
-
- "vtrn.u8 q8, q9 \n" // _src45t_r
-
- "pld [%2, #128] \n"
- "vld2.u8 {d22-d23}, [%2], %10\n"
-
- "vtrn.u8 q10, q11 \n" // _src67t_r
-
- "sub %1, %1, %10, lsl #2 \n" // restore src0
-
- "vtrn.u16 q0, q2 \n" // _src02tt_r
-
- "sub %2, %2, %10, lsl #2 \n" // restore src1
-
- "vtrn.u16 q1, q3 \n" // _src13tt_r
-
- "add %1, #16 \n" // src0 += 16
-
- "vtrn.u16 q8, q10 \n" // _src46tt_r
-
- "add %2, #16 \n" // src1 += 16
-
- "vtrn.u16 q9, q11 \n" // _src57tt_r
-
- "vtrn.u32 q0, q8 \n" // _src04ttt_r
-
- "vtrn.u32 q1, q9 \n" // _src15ttt_r
- "vst2.u8 {d0-d1}, [%3], %11 \n"
-
- "vtrn.u32 q2, q10 \n" // _src26ttt_r
- "vst2.u8 {d2-d3}, [%4], %11 \n"
-
- "vtrn.u32 q3, q11 \n" // _src37ttt_r
- "vst2.u8 {d4-d5}, [%3], %11 \n"
-
- "subs %0, #1 \n"
-
- "vst2.u8 {d6-d7}, [%4], %11 \n"
- "vst2.u8 {d16-d17}, [%3], %11\n"
- "vst2.u8 {d18-d19}, [%4], %11\n"
- "vst2.u8 {d20-d21}, [%3], %11\n"
- "vst2.u8 {d22-d23}, [%4], %11\n"
-
- "bne 0b \n"
- : "=r"(nn), // %0
- "=r"(src0), // %1
- "=r"(src1), // %2
- "=r"(dst0), // %3
- "=r"(dst1) // %4
- : "0"(nn),
- "1"(src0),
- "2"(src1),
- "3"(dst0),
- "4"(dst1),
- "r"(src_step), // %10
- "r"(dst_step) // %11
- : "cc", "memory", "q0", "q1", "q2", "q3", "q8", "q9", "q10", "q11");
- }
- #endif // __aarch64__
- for (; remain > 0; remain--)
- {
- dst0[0] = src0[0];
- dst0[1] = src0[1];
- dst0[2] = src1[0];
- dst0[3] = src1[1];
- dst0[4] = src0[0 + src_step];
- dst0[5] = src0[1 + src_step];
- dst0[6] = src1[0 + src_step];
- dst0[7] = src1[1 + src_step];
- dst0[8] = src0[0 + 2 * src_step];
- dst0[9] = src0[1 + 2 * src_step];
- dst0[10] = src1[0 + 2 * src_step];
- dst0[11] = src1[1 + 2 * src_step];
- dst0[12] = src0[0 + 3 * src_step];
- dst0[13] = src0[1 + 3 * src_step];
- dst0[14] = src1[0 + 3 * src_step];
- dst0[15] = src1[1 + 3 * src_step];
-
- src0 += 2;
- src1 += 2;
-
- dst0 += stride;
- }
-
- src0 += srcwgap + 7 * srcstride;
- }
- #endif // __ARM_NEON
- for (; y < srch; y++)
- {
- unsigned char* dst0 = dst + y * 2;
-
- int x = 0;
- for (; x < srcw; x++)
- {
- dst0[0] = src0[0];
- dst0[1] = src0[1];
-
- src0 += 2;
- dst0 += stride;
- }
-
- src0 += srcwgap;
- }
- }
-
- static void kanna_rotate_5_c3(const unsigned char* src, int srcw, int srch, int srcstride, unsigned char* dst, int /*w*/, int /*h*/, int stride)
- {
- const int srcwgap = srcstride - srcw * 3;
-
- const unsigned char* src0 = src;
-
- int y = 0;
- #if __ARM_NEON
- for (; y + 7 < srch; y += 8)
- {
- const unsigned char* src1 = src0 + srcstride;
-
- unsigned char* dst0 = dst + y * 3;
- unsigned char* dst1 = dst + y * 3 + stride;
-
- int src_step = 2 * srcstride;
- int dst_step = 2 * stride;
-
- int nn = srcw >> 3;
- int remain = srcw - (nn << 3);
-
- #if !NCNN_GNU_INLINE_ASM || __aarch64__
- for (; nn > 0; nn--)
- {
- uint8x8x3_t _src0 = vld3_u8(src0);
- uint8x8x3_t _src1 = vld3_u8(src1);
-
- uint8x8x3_t _src2 = vld3_u8(src0 + src_step);
- uint8x8x3_t _src3 = vld3_u8(src1 + src_step);
-
- uint8x8x3_t _src4 = vld3_u8(src0 + 2 * src_step);
- uint8x8x3_t _src5 = vld3_u8(src1 + 2 * src_step);
-
- uint8x8x3_t _src6 = vld3_u8(src0 + 3 * src_step);
- uint8x8x3_t _src7 = vld3_u8(src1 + 3 * src_step);
-
- uint8x8x2_t _src01t_r = vtrn_u8(_src0.val[0], _src1.val[0]);
- uint8x8x2_t _src23t_r = vtrn_u8(_src2.val[0], _src3.val[0]);
- uint8x8x2_t _src45t_r = vtrn_u8(_src4.val[0], _src5.val[0]);
- uint8x8x2_t _src67t_r = vtrn_u8(_src6.val[0], _src7.val[0]);
-
- uint8x8x2_t _src01t_g = vtrn_u8(_src0.val[1], _src1.val[1]);
- uint8x8x2_t _src23t_g = vtrn_u8(_src2.val[1], _src3.val[1]);
- uint8x8x2_t _src45t_g = vtrn_u8(_src4.val[1], _src5.val[1]);
- uint8x8x2_t _src67t_g = vtrn_u8(_src6.val[1], _src7.val[1]);
-
- uint8x8x2_t _src01t_b = vtrn_u8(_src0.val[2], _src1.val[2]);
- uint8x8x2_t _src23t_b = vtrn_u8(_src2.val[2], _src3.val[2]);
- uint8x8x2_t _src45t_b = vtrn_u8(_src4.val[2], _src5.val[2]);
- uint8x8x2_t _src67t_b = vtrn_u8(_src6.val[2], _src7.val[2]);
-
- uint16x4x2_t _src02tt_r = vtrn_u16(vreinterpret_u16_u8(_src01t_r.val[0]), vreinterpret_u16_u8(_src23t_r.val[0]));
- uint16x4x2_t _src13tt_r = vtrn_u16(vreinterpret_u16_u8(_src01t_r.val[1]), vreinterpret_u16_u8(_src23t_r.val[1]));
- uint16x4x2_t _src46tt_r = vtrn_u16(vreinterpret_u16_u8(_src45t_r.val[0]), vreinterpret_u16_u8(_src67t_r.val[0]));
- uint16x4x2_t _src57tt_r = vtrn_u16(vreinterpret_u16_u8(_src45t_r.val[1]), vreinterpret_u16_u8(_src67t_r.val[1]));
-
- uint16x4x2_t _src02tt_g = vtrn_u16(vreinterpret_u16_u8(_src01t_g.val[0]), vreinterpret_u16_u8(_src23t_g.val[0]));
- uint16x4x2_t _src13tt_g = vtrn_u16(vreinterpret_u16_u8(_src01t_g.val[1]), vreinterpret_u16_u8(_src23t_g.val[1]));
- uint16x4x2_t _src46tt_g = vtrn_u16(vreinterpret_u16_u8(_src45t_g.val[0]), vreinterpret_u16_u8(_src67t_g.val[0]));
- uint16x4x2_t _src57tt_g = vtrn_u16(vreinterpret_u16_u8(_src45t_g.val[1]), vreinterpret_u16_u8(_src67t_g.val[1]));
-
- uint16x4x2_t _src02tt_b = vtrn_u16(vreinterpret_u16_u8(_src01t_b.val[0]), vreinterpret_u16_u8(_src23t_b.val[0]));
- uint16x4x2_t _src13tt_b = vtrn_u16(vreinterpret_u16_u8(_src01t_b.val[1]), vreinterpret_u16_u8(_src23t_b.val[1]));
- uint16x4x2_t _src46tt_b = vtrn_u16(vreinterpret_u16_u8(_src45t_b.val[0]), vreinterpret_u16_u8(_src67t_b.val[0]));
- uint16x4x2_t _src57tt_b = vtrn_u16(vreinterpret_u16_u8(_src45t_b.val[1]), vreinterpret_u16_u8(_src67t_b.val[1]));
-
- uint32x2x2_t _src04ttt_r = vtrn_u32(vreinterpret_u32_u16(_src02tt_r.val[0]), vreinterpret_u32_u16(_src46tt_r.val[0]));
- uint32x2x2_t _src15ttt_r = vtrn_u32(vreinterpret_u32_u16(_src13tt_r.val[0]), vreinterpret_u32_u16(_src57tt_r.val[0]));
- uint32x2x2_t _src26ttt_r = vtrn_u32(vreinterpret_u32_u16(_src02tt_r.val[1]), vreinterpret_u32_u16(_src46tt_r.val[1]));
- uint32x2x2_t _src37ttt_r = vtrn_u32(vreinterpret_u32_u16(_src13tt_r.val[1]), vreinterpret_u32_u16(_src57tt_r.val[1]));
-
- uint32x2x2_t _src04ttt_g = vtrn_u32(vreinterpret_u32_u16(_src02tt_g.val[0]), vreinterpret_u32_u16(_src46tt_g.val[0]));
- uint32x2x2_t _src15ttt_g = vtrn_u32(vreinterpret_u32_u16(_src13tt_g.val[0]), vreinterpret_u32_u16(_src57tt_g.val[0]));
- uint32x2x2_t _src26ttt_g = vtrn_u32(vreinterpret_u32_u16(_src02tt_g.val[1]), vreinterpret_u32_u16(_src46tt_g.val[1]));
- uint32x2x2_t _src37ttt_g = vtrn_u32(vreinterpret_u32_u16(_src13tt_g.val[1]), vreinterpret_u32_u16(_src57tt_g.val[1]));
-
- uint32x2x2_t _src04ttt_b = vtrn_u32(vreinterpret_u32_u16(_src02tt_b.val[0]), vreinterpret_u32_u16(_src46tt_b.val[0]));
- uint32x2x2_t _src15ttt_b = vtrn_u32(vreinterpret_u32_u16(_src13tt_b.val[0]), vreinterpret_u32_u16(_src57tt_b.val[0]));
- uint32x2x2_t _src26ttt_b = vtrn_u32(vreinterpret_u32_u16(_src02tt_b.val[1]), vreinterpret_u32_u16(_src46tt_b.val[1]));
- uint32x2x2_t _src37ttt_b = vtrn_u32(vreinterpret_u32_u16(_src13tt_b.val[1]), vreinterpret_u32_u16(_src57tt_b.val[1]));
-
- uint8x8x3_t _dst0;
- uint8x8x3_t _dst1;
- uint8x8x3_t _dst2;
- uint8x8x3_t _dst3;
- uint8x8x3_t _dst4;
- uint8x8x3_t _dst5;
- uint8x8x3_t _dst6;
- uint8x8x3_t _dst7;
-
- _dst0.val[0] = vreinterpret_u8_u32(_src04ttt_r.val[0]);
- _dst1.val[0] = vreinterpret_u8_u32(_src15ttt_r.val[0]);
- _dst2.val[0] = vreinterpret_u8_u32(_src26ttt_r.val[0]);
- _dst3.val[0] = vreinterpret_u8_u32(_src37ttt_r.val[0]);
- _dst4.val[0] = vreinterpret_u8_u32(_src04ttt_r.val[1]);
- _dst5.val[0] = vreinterpret_u8_u32(_src15ttt_r.val[1]);
- _dst6.val[0] = vreinterpret_u8_u32(_src26ttt_r.val[1]);
- _dst7.val[0] = vreinterpret_u8_u32(_src37ttt_r.val[1]);
-
- _dst0.val[1] = vreinterpret_u8_u32(_src04ttt_g.val[0]);
- _dst1.val[1] = vreinterpret_u8_u32(_src15ttt_g.val[0]);
- _dst2.val[1] = vreinterpret_u8_u32(_src26ttt_g.val[0]);
- _dst3.val[1] = vreinterpret_u8_u32(_src37ttt_g.val[0]);
- _dst4.val[1] = vreinterpret_u8_u32(_src04ttt_g.val[1]);
- _dst5.val[1] = vreinterpret_u8_u32(_src15ttt_g.val[1]);
- _dst6.val[1] = vreinterpret_u8_u32(_src26ttt_g.val[1]);
- _dst7.val[1] = vreinterpret_u8_u32(_src37ttt_g.val[1]);
-
- _dst0.val[2] = vreinterpret_u8_u32(_src04ttt_b.val[0]);
- _dst1.val[2] = vreinterpret_u8_u32(_src15ttt_b.val[0]);
- _dst2.val[2] = vreinterpret_u8_u32(_src26ttt_b.val[0]);
- _dst3.val[2] = vreinterpret_u8_u32(_src37ttt_b.val[0]);
- _dst4.val[2] = vreinterpret_u8_u32(_src04ttt_b.val[1]);
- _dst5.val[2] = vreinterpret_u8_u32(_src15ttt_b.val[1]);
- _dst6.val[2] = vreinterpret_u8_u32(_src26ttt_b.val[1]);
- _dst7.val[2] = vreinterpret_u8_u32(_src37ttt_b.val[1]);
-
- vst3_u8(dst0, _dst0);
- vst3_u8(dst1, _dst1);
- vst3_u8(dst0 + dst_step, _dst2);
- vst3_u8(dst1 + dst_step, _dst3);
- vst3_u8(dst0 + 2 * dst_step, _dst4);
- vst3_u8(dst1 + 2 * dst_step, _dst5);
- vst3_u8(dst0 + 3 * dst_step, _dst6);
- vst3_u8(dst1 + 3 * dst_step, _dst7);
-
- src0 += 3 * 8;
- src1 += 3 * 8;
-
- dst0 += 4 * dst_step;
- dst1 += 4 * dst_step;
- }
- #else
- if (nn > 0)
- {
- asm volatile(
- "0: \n"
- "pld [%1, #192] \n"
- "vld3.u8 {d0-d2}, [%1], %10 \n"
-
- "pld [%2, #192] \n"
- "vld3.u8 {d4-d6}, [%2], %10 \n"
-
- "pld [%1, #192] \n"
- "vld3.u8 {d8-d10}, [%1], %10 \n"
-
- "vtrn.u8 q0, q2 \n" // _src01t_r
- "vtrn.u8 d2, d6 \n"
-
- "pld [%2, #192] \n"
- "vld3.u8 {d12-d14}, [%2], %10\n"
-
- "pld [%1, #192] \n"
- "vld3.u8 {d16-d18}, [%1], %10\n"
-
- "vtrn.u8 q4, q6 \n" // _src23t_r
- "vtrn.u8 d10, d14 \n"
-
- "pld [%2, #192] \n"
- "vld3.u8 {d20-d22}, [%2], %10\n"
-
- "pld [%1, #192] \n"
- "vld3.u8 {d24-d26}, [%1], %10\n"
-
- "vtrn.u8 q8, q10 \n" // _src45t_r
- "vtrn.u8 d18, d22 \n"
-
- "pld [%2, #192] \n"
- "vld3.u8 {d28-d30}, [%2], %10\n"
-
- "vtrn.u8 q12, q14 \n" // _src67t_r
- "vtrn.u8 d26, d30 \n"
-
- "sub %1, %1, %10, lsl #2 \n" // restore src0
-
- "vtrn.u16 q0, q4 \n" // _src02tt_r
- "vtrn.u16 d2, d10 \n"
-
- "sub %2, %2, %10, lsl #2 \n" // restore src1
-
- "vtrn.u16 q2, q6 \n" // _src13tt_r
- "vtrn.u16 d6, d14 \n"
-
- "add %1, #24 \n" // src0 += 24
-
- "vtrn.u16 q8, q12 \n" // _src46tt_r
- "vtrn.u16 d18, d26 \n"
-
- "add %2, #24 \n" // src1 += 24
-
- "vtrn.u16 q10, q14 \n" // _src57tt_r
- "vtrn.u16 d22, d30 \n"
-
- "vtrn.u32 q0, q8 \n" // _src04ttt_r
- "vtrn.u32 d2, d18 \n"
-
- "vtrn.u32 q2, q10 \n" // _src15ttt_r
- "vst3.u8 {d0-d2}, [%3], %11 \n"
- "vtrn.u32 d6, d22 \n"
-
- "vtrn.u32 q4, q12 \n" // _src26ttt_r
- "vst3.u8 {d4-d6}, [%4], %11 \n"
- "vtrn.u32 d10, d26 \n"
-
- "vtrn.u32 q6, q14 \n" // _src37ttt_r
- "vst3.u8 {d8-d10}, [%3], %11 \n"
- "vtrn.u32 d14, d30 \n"
-
- "subs %0, #1 \n"
-
- "vst3.u8 {d16-d18}, [%3], %11\n"
- "vst3.u8 {d12-d14}, [%4], %11\n"
- "vst3.u8 {d20-d22}, [%4], %11\n"
- "vst3.u8 {d24-d26}, [%3], %11\n"
- "vst3.u8 {d28-d30}, [%4], %11\n"
-
- "bne 0b \n"
- : "=r"(nn), // %0
- "=r"(src0), // %1
- "=r"(src1), // %2
- "=r"(dst0), // %3
- "=r"(dst1) // %4
- : "0"(nn),
- "1"(src0),
- "2"(src1),
- "3"(dst0),
- "4"(dst1),
- "r"(src_step), // %10
- "r"(dst_step) // %11
- : "cc", "memory", "q0", "q1", "q2", "q3", "q4", "q5", "q6", "q7", "q8", "q9", "q10", "q11", "q12", "q13", "q14", "q15");
- }
- #endif // __aarch64__
- for (; remain > 0; remain--)
- {
- dst0[0] = src0[0];
- dst0[1] = src0[1];
- dst0[2] = src0[2];
- dst0[3] = src1[0];
- dst0[4] = src1[1];
- dst0[5] = src1[2];
- dst0[6] = src0[0 + src_step];
- dst0[7] = src0[1 + src_step];
- dst0[8] = src0[2 + src_step];
- dst0[9] = src1[0 + src_step];
- dst0[10] = src1[1 + src_step];
- dst0[11] = src1[2 + src_step];
- dst0[12] = src0[0 + 2 * src_step];
- dst0[13] = src0[1 + 2 * src_step];
- dst0[14] = src0[2 + 2 * src_step];
- dst0[15] = src1[0 + 2 * src_step];
- dst0[16] = src1[1 + 2 * src_step];
- dst0[17] = src1[2 + 2 * src_step];
- dst0[18] = src0[0 + 3 * src_step];
- dst0[19] = src0[1 + 3 * src_step];
- dst0[20] = src0[2 + 3 * src_step];
- dst0[21] = src1[0 + 3 * src_step];
- dst0[22] = src1[1 + 3 * src_step];
- dst0[23] = src1[2 + 3 * src_step];
-
- src0 += 3;
- src1 += 3;
-
- dst0 += stride;
- }
-
- src0 += srcwgap + 7 * srcstride;
- }
- #endif // __ARM_NEON
- for (; y < srch; y++)
- {
- unsigned char* dst0 = dst + y * 3;
-
- int x = 0;
- for (; x < srcw; x++)
- {
- dst0[0] = src0[0];
- dst0[1] = src0[1];
- dst0[2] = src0[2];
-
- src0 += 3;
- dst0 += stride;
- }
-
- src0 += srcwgap;
- }
- }
-
- static void kanna_rotate_5_c4(const unsigned char* src, int srcw, int srch, int srcstride, unsigned char* dst, int /*w*/, int /*h*/, int stride)
- {
- const int srcwgap = srcstride - srcw * 4;
-
- const unsigned char* src0 = src;
-
- int y = 0;
- #if __ARM_NEON
- for (; y + 7 < srch; y += 8)
- {
- const unsigned char* src1 = src0 + srcstride;
-
- unsigned char* dst0 = dst + y * 4;
- unsigned char* dst1 = dst + y * 4 + stride;
-
- int src_step = 2 * srcstride;
- int dst_step = 2 * stride;
-
- int nn = srcw >> 3;
- int remain = srcw - (nn << 3);
-
- #if !NCNN_GNU_INLINE_ASM || __aarch64__
- for (; nn > 0; nn--)
- {
- uint8x8x4_t _src0 = vld4_u8(src0);
- uint8x8x4_t _src1 = vld4_u8(src1);
-
- uint8x8x4_t _src2 = vld4_u8(src0 + src_step);
- uint8x8x4_t _src3 = vld4_u8(src1 + src_step);
-
- uint8x8x4_t _src4 = vld4_u8(src0 + 2 * src_step);
- uint8x8x4_t _src5 = vld4_u8(src1 + 2 * src_step);
-
- uint8x8x4_t _src6 = vld4_u8(src0 + 3 * src_step);
- uint8x8x4_t _src7 = vld4_u8(src1 + 3 * src_step);
-
- uint8x8x2_t _src01t_r = vtrn_u8(_src0.val[0], _src1.val[0]);
- uint8x8x2_t _src23t_r = vtrn_u8(_src2.val[0], _src3.val[0]);
- uint8x8x2_t _src45t_r = vtrn_u8(_src4.val[0], _src5.val[0]);
- uint8x8x2_t _src67t_r = vtrn_u8(_src6.val[0], _src7.val[0]);
-
- uint8x8x2_t _src01t_g = vtrn_u8(_src0.val[1], _src1.val[1]);
- uint8x8x2_t _src23t_g = vtrn_u8(_src2.val[1], _src3.val[1]);
- uint8x8x2_t _src45t_g = vtrn_u8(_src4.val[1], _src5.val[1]);
- uint8x8x2_t _src67t_g = vtrn_u8(_src6.val[1], _src7.val[1]);
-
- uint8x8x2_t _src01t_b = vtrn_u8(_src0.val[2], _src1.val[2]);
- uint8x8x2_t _src23t_b = vtrn_u8(_src2.val[2], _src3.val[2]);
- uint8x8x2_t _src45t_b = vtrn_u8(_src4.val[2], _src5.val[2]);
- uint8x8x2_t _src67t_b = vtrn_u8(_src6.val[2], _src7.val[2]);
-
- uint8x8x2_t _src01t_a = vtrn_u8(_src0.val[3], _src1.val[3]);
- uint8x8x2_t _src23t_a = vtrn_u8(_src2.val[3], _src3.val[3]);
- uint8x8x2_t _src45t_a = vtrn_u8(_src4.val[3], _src5.val[3]);
- uint8x8x2_t _src67t_a = vtrn_u8(_src6.val[3], _src7.val[3]);
-
- uint16x4x2_t _src02tt_r = vtrn_u16(vreinterpret_u16_u8(_src01t_r.val[0]), vreinterpret_u16_u8(_src23t_r.val[0]));
- uint16x4x2_t _src13tt_r = vtrn_u16(vreinterpret_u16_u8(_src01t_r.val[1]), vreinterpret_u16_u8(_src23t_r.val[1]));
- uint16x4x2_t _src46tt_r = vtrn_u16(vreinterpret_u16_u8(_src45t_r.val[0]), vreinterpret_u16_u8(_src67t_r.val[0]));
- uint16x4x2_t _src57tt_r = vtrn_u16(vreinterpret_u16_u8(_src45t_r.val[1]), vreinterpret_u16_u8(_src67t_r.val[1]));
-
- uint16x4x2_t _src02tt_g = vtrn_u16(vreinterpret_u16_u8(_src01t_g.val[0]), vreinterpret_u16_u8(_src23t_g.val[0]));
- uint16x4x2_t _src13tt_g = vtrn_u16(vreinterpret_u16_u8(_src01t_g.val[1]), vreinterpret_u16_u8(_src23t_g.val[1]));
- uint16x4x2_t _src46tt_g = vtrn_u16(vreinterpret_u16_u8(_src45t_g.val[0]), vreinterpret_u16_u8(_src67t_g.val[0]));
- uint16x4x2_t _src57tt_g = vtrn_u16(vreinterpret_u16_u8(_src45t_g.val[1]), vreinterpret_u16_u8(_src67t_g.val[1]));
-
- uint16x4x2_t _src02tt_b = vtrn_u16(vreinterpret_u16_u8(_src01t_b.val[0]), vreinterpret_u16_u8(_src23t_b.val[0]));
- uint16x4x2_t _src13tt_b = vtrn_u16(vreinterpret_u16_u8(_src01t_b.val[1]), vreinterpret_u16_u8(_src23t_b.val[1]));
- uint16x4x2_t _src46tt_b = vtrn_u16(vreinterpret_u16_u8(_src45t_b.val[0]), vreinterpret_u16_u8(_src67t_b.val[0]));
- uint16x4x2_t _src57tt_b = vtrn_u16(vreinterpret_u16_u8(_src45t_b.val[1]), vreinterpret_u16_u8(_src67t_b.val[1]));
-
- uint16x4x2_t _src02tt_a = vtrn_u16(vreinterpret_u16_u8(_src01t_a.val[0]), vreinterpret_u16_u8(_src23t_a.val[0]));
- uint16x4x2_t _src13tt_a = vtrn_u16(vreinterpret_u16_u8(_src01t_a.val[1]), vreinterpret_u16_u8(_src23t_a.val[1]));
- uint16x4x2_t _src46tt_a = vtrn_u16(vreinterpret_u16_u8(_src45t_a.val[0]), vreinterpret_u16_u8(_src67t_a.val[0]));
- uint16x4x2_t _src57tt_a = vtrn_u16(vreinterpret_u16_u8(_src45t_a.val[1]), vreinterpret_u16_u8(_src67t_a.val[1]));
-
- uint32x2x2_t _src04ttt_r = vtrn_u32(vreinterpret_u32_u16(_src02tt_r.val[0]), vreinterpret_u32_u16(_src46tt_r.val[0]));
- uint32x2x2_t _src15ttt_r = vtrn_u32(vreinterpret_u32_u16(_src13tt_r.val[0]), vreinterpret_u32_u16(_src57tt_r.val[0]));
- uint32x2x2_t _src26ttt_r = vtrn_u32(vreinterpret_u32_u16(_src02tt_r.val[1]), vreinterpret_u32_u16(_src46tt_r.val[1]));
- uint32x2x2_t _src37ttt_r = vtrn_u32(vreinterpret_u32_u16(_src13tt_r.val[1]), vreinterpret_u32_u16(_src57tt_r.val[1]));
-
- uint32x2x2_t _src04ttt_g = vtrn_u32(vreinterpret_u32_u16(_src02tt_g.val[0]), vreinterpret_u32_u16(_src46tt_g.val[0]));
- uint32x2x2_t _src15ttt_g = vtrn_u32(vreinterpret_u32_u16(_src13tt_g.val[0]), vreinterpret_u32_u16(_src57tt_g.val[0]));
- uint32x2x2_t _src26ttt_g = vtrn_u32(vreinterpret_u32_u16(_src02tt_g.val[1]), vreinterpret_u32_u16(_src46tt_g.val[1]));
- uint32x2x2_t _src37ttt_g = vtrn_u32(vreinterpret_u32_u16(_src13tt_g.val[1]), vreinterpret_u32_u16(_src57tt_g.val[1]));
-
- uint32x2x2_t _src04ttt_b = vtrn_u32(vreinterpret_u32_u16(_src02tt_b.val[0]), vreinterpret_u32_u16(_src46tt_b.val[0]));
- uint32x2x2_t _src15ttt_b = vtrn_u32(vreinterpret_u32_u16(_src13tt_b.val[0]), vreinterpret_u32_u16(_src57tt_b.val[0]));
- uint32x2x2_t _src26ttt_b = vtrn_u32(vreinterpret_u32_u16(_src02tt_b.val[1]), vreinterpret_u32_u16(_src46tt_b.val[1]));
- uint32x2x2_t _src37ttt_b = vtrn_u32(vreinterpret_u32_u16(_src13tt_b.val[1]), vreinterpret_u32_u16(_src57tt_b.val[1]));
-
- uint32x2x2_t _src04ttt_a = vtrn_u32(vreinterpret_u32_u16(_src02tt_a.val[0]), vreinterpret_u32_u16(_src46tt_a.val[0]));
- uint32x2x2_t _src15ttt_a = vtrn_u32(vreinterpret_u32_u16(_src13tt_a.val[0]), vreinterpret_u32_u16(_src57tt_a.val[0]));
- uint32x2x2_t _src26ttt_a = vtrn_u32(vreinterpret_u32_u16(_src02tt_a.val[1]), vreinterpret_u32_u16(_src46tt_a.val[1]));
- uint32x2x2_t _src37ttt_a = vtrn_u32(vreinterpret_u32_u16(_src13tt_a.val[1]), vreinterpret_u32_u16(_src57tt_a.val[1]));
-
- uint8x8x4_t _dst0;
- uint8x8x4_t _dst1;
- uint8x8x4_t _dst2;
- uint8x8x4_t _dst3;
- uint8x8x4_t _dst4;
- uint8x8x4_t _dst5;
- uint8x8x4_t _dst6;
- uint8x8x4_t _dst7;
-
- _dst0.val[0] = vreinterpret_u8_u32(_src04ttt_r.val[0]);
- _dst1.val[0] = vreinterpret_u8_u32(_src15ttt_r.val[0]);
- _dst2.val[0] = vreinterpret_u8_u32(_src26ttt_r.val[0]);
- _dst3.val[0] = vreinterpret_u8_u32(_src37ttt_r.val[0]);
- _dst4.val[0] = vreinterpret_u8_u32(_src04ttt_r.val[1]);
- _dst5.val[0] = vreinterpret_u8_u32(_src15ttt_r.val[1]);
- _dst6.val[0] = vreinterpret_u8_u32(_src26ttt_r.val[1]);
- _dst7.val[0] = vreinterpret_u8_u32(_src37ttt_r.val[1]);
-
- _dst0.val[1] = vreinterpret_u8_u32(_src04ttt_g.val[0]);
- _dst1.val[1] = vreinterpret_u8_u32(_src15ttt_g.val[0]);
- _dst2.val[1] = vreinterpret_u8_u32(_src26ttt_g.val[0]);
- _dst3.val[1] = vreinterpret_u8_u32(_src37ttt_g.val[0]);
- _dst4.val[1] = vreinterpret_u8_u32(_src04ttt_g.val[1]);
- _dst5.val[1] = vreinterpret_u8_u32(_src15ttt_g.val[1]);
- _dst6.val[1] = vreinterpret_u8_u32(_src26ttt_g.val[1]);
- _dst7.val[1] = vreinterpret_u8_u32(_src37ttt_g.val[1]);
-
- _dst0.val[2] = vreinterpret_u8_u32(_src04ttt_b.val[0]);
- _dst1.val[2] = vreinterpret_u8_u32(_src15ttt_b.val[0]);
- _dst2.val[2] = vreinterpret_u8_u32(_src26ttt_b.val[0]);
- _dst3.val[2] = vreinterpret_u8_u32(_src37ttt_b.val[0]);
- _dst4.val[2] = vreinterpret_u8_u32(_src04ttt_b.val[1]);
- _dst5.val[2] = vreinterpret_u8_u32(_src15ttt_b.val[1]);
- _dst6.val[2] = vreinterpret_u8_u32(_src26ttt_b.val[1]);
- _dst7.val[2] = vreinterpret_u8_u32(_src37ttt_b.val[1]);
-
- _dst0.val[3] = vreinterpret_u8_u32(_src04ttt_a.val[0]);
- _dst1.val[3] = vreinterpret_u8_u32(_src15ttt_a.val[0]);
- _dst2.val[3] = vreinterpret_u8_u32(_src26ttt_a.val[0]);
- _dst3.val[3] = vreinterpret_u8_u32(_src37ttt_a.val[0]);
- _dst4.val[3] = vreinterpret_u8_u32(_src04ttt_a.val[1]);
- _dst5.val[3] = vreinterpret_u8_u32(_src15ttt_a.val[1]);
- _dst6.val[3] = vreinterpret_u8_u32(_src26ttt_a.val[1]);
- _dst7.val[3] = vreinterpret_u8_u32(_src37ttt_a.val[1]);
-
- vst4_u8(dst0, _dst0);
- vst4_u8(dst1, _dst1);
- vst4_u8(dst0 + dst_step, _dst2);
- vst4_u8(dst1 + dst_step, _dst3);
- vst4_u8(dst0 + 2 * dst_step, _dst4);
- vst4_u8(dst1 + 2 * dst_step, _dst5);
- vst4_u8(dst0 + 3 * dst_step, _dst6);
- vst4_u8(dst1 + 3 * dst_step, _dst7);
-
- src0 += 4 * 8;
- src1 += 4 * 8;
-
- dst0 += 4 * dst_step;
- dst1 += 4 * dst_step;
- }
- #else
- if (nn > 0)
- {
- asm volatile(
- "0: \n"
- "pld [%1, #256] \n"
- "vld4.u8 {d0-d3}, [%1], %10 \n"
-
- "pld [%2, #256] \n"
- "vld4.u8 {d4-d7}, [%2], %10 \n"
-
- "pld [%1, #256] \n"
- "vld4.u8 {d8-d11}, [%1], %10 \n"
-
- "vtrn.u8 q0, q2 \n" // _src01t_r
- "vtrn.u8 q1, q3 \n"
-
- "pld [%2, #256] \n"
- "vld4.u8 {d12-d15}, [%2], %10\n"
-
- "pld [%1, #256] \n"
- "vld4.u8 {d16-d19}, [%1], %10\n"
-
- "vtrn.u8 q4, q6 \n" // _src23t_r
- "vtrn.u8 q5, q7 \n"
-
- "pld [%2, #256] \n"
- "vld4.u8 {d20-d23}, [%2], %10\n"
-
- "pld [%1, #256] \n"
- "vld4.u8 {d24-d27}, [%1], %10\n"
-
- "vtrn.u8 q8, q10 \n" // _src45t_r
- "vtrn.u8 q9, q11 \n"
-
- "pld [%2, #256] \n"
- "vld4.u8 {d28-d31}, [%2], %10\n"
-
- "vtrn.u8 q12, q14 \n" // _src67t_r
- "vtrn.u8 q13, q15 \n"
-
- "sub %1, %1, %10, lsl #2 \n" // restore src0
-
- "vtrn.u16 q0, q4 \n" // _src02tt_r
- "vtrn.u16 q1, q5 \n"
-
- "sub %2, %2, %10, lsl #2 \n" // restore src1
-
- "vtrn.u16 q2, q6 \n" // _src13tt_r
- "vtrn.u16 q3, q7 \n"
-
- "add %1, #32 \n" // src0 += 32
-
- "vtrn.u16 q8, q12 \n" // _src46tt_r
- "vtrn.u16 q9, q13 \n"
-
- "add %2, #32 \n" // src1 += 32
-
- "vtrn.u16 q10, q14 \n" // _src57tt_r
- "vtrn.u16 q11, q15 \n"
-
- "vtrn.u32 q0, q8 \n" // _src04ttt_r
- "vtrn.u32 q1, q9 \n"
-
- "vtrn.u32 q2, q10 \n" // _src15ttt_r
- "vst4.u8 {d0-d3}, [%3], %11 \n"
- "vtrn.u32 q3, q11 \n"
-
- "vtrn.u32 q4, q12 \n" // _src26ttt_r
- "vst4.u8 {d4-d7}, [%4], %11 \n"
- "vtrn.u32 q5, q13 \n"
-
- "vtrn.u32 q6, q14 \n" // _src37ttt_r
- "vst4.u8 {d8-d11}, [%3], %11 \n"
- "vtrn.u32 q7, q15 \n"
-
- "subs %0, #1 \n"
-
- "vst4.u8 {d16-d19}, [%3], %11\n"
- "vst4.u8 {d12-d15}, [%4], %11\n"
- "vst4.u8 {d20-d23}, [%4], %11\n"
- "vst4.u8 {d24-d27}, [%3], %11\n"
- "vst4.u8 {d28-d31}, [%4], %11\n"
-
- "bne 0b \n"
- : "=r"(nn), // %0
- "=r"(src0), // %1
- "=r"(src1), // %2
- "=r"(dst0), // %3
- "=r"(dst1) // %4
- : "0"(nn),
- "1"(src0),
- "2"(src1),
- "3"(dst0),
- "4"(dst1),
- "r"(src_step), // %10
- "r"(dst_step) // %11
- : "cc", "memory", "q0", "q1", "q2", "q3", "q4", "q5", "q6", "q7", "q8", "q9", "q10", "q11", "q12", "q13", "q14", "q15");
- }
- #endif // __aarch64__
- for (; remain > 0; remain--)
- {
- dst0[0] = src0[0];
- dst0[1] = src0[1];
- dst0[2] = src0[2];
- dst0[3] = src0[3];
- dst0[4] = src1[0];
- dst0[5] = src1[1];
- dst0[6] = src1[2];
- dst0[7] = src1[3];
- dst0[8] = src0[0 + src_step];
- dst0[9] = src0[1 + src_step];
- dst0[10] = src0[2 + src_step];
- dst0[11] = src0[3 + src_step];
- dst0[12] = src1[0 + src_step];
- dst0[13] = src1[1 + src_step];
- dst0[14] = src1[2 + src_step];
- dst0[15] = src1[3 + src_step];
- dst0[16] = src0[0 + 2 * src_step];
- dst0[17] = src0[1 + 2 * src_step];
- dst0[18] = src0[2 + 2 * src_step];
- dst0[19] = src0[3 + 2 * src_step];
- dst0[20] = src1[0 + 2 * src_step];
- dst0[21] = src1[1 + 2 * src_step];
- dst0[22] = src1[2 + 2 * src_step];
- dst0[23] = src1[3 + 2 * src_step];
- dst0[24] = src0[0 + 3 * src_step];
- dst0[25] = src0[1 + 3 * src_step];
- dst0[26] = src0[2 + 3 * src_step];
- dst0[27] = src0[3 + 3 * src_step];
- dst0[28] = src1[0 + 3 * src_step];
- dst0[29] = src1[1 + 3 * src_step];
- dst0[30] = src1[2 + 3 * src_step];
- dst0[31] = src1[3 + 3 * src_step];
-
- src0 += 4;
- src1 += 4;
-
- dst0 += stride;
- }
-
- src0 += srcwgap + 7 * srcstride;
- }
- #endif // __ARM_NEON
- for (; y < srch; y++)
- {
- unsigned char* dst0 = dst + y * 4;
-
- int x = 0;
- for (; x < srcw; x++)
- {
- dst0[0] = src0[0];
- dst0[1] = src0[1];
- dst0[2] = src0[2];
- dst0[3] = src0[3];
-
- src0 += 4;
- dst0 += stride;
- }
-
- src0 += srcwgap;
- }
- }
-
- static void kanna_rotate_6_c1(const unsigned char* src, int srcw, int srch, int srcstride, unsigned char* dst, int w, int /*h*/, int stride)
- {
- const int srcwgap = srcstride - srcw;
-
- // point to the last dst pixel in row
- unsigned char* dstend = dst + w;
-
- const unsigned char* src0 = src;
-
- int y = 0;
- #if __ARM_NEON
- for (; y + 7 < srch; y += 8)
- {
- const unsigned char* src1 = src0 + srcstride;
-
- unsigned char* dst0 = dstend - y - 8;
- unsigned char* dst1 = dstend - y - 8 + stride;
-
- int src_step = 2 * srcstride;
- int dst_step = 2 * stride;
-
- int nn = srcw >> 3;
- int remain = srcw - (nn << 3);
-
- #if !NCNN_GNU_INLINE_ASM || __aarch64__
- for (; nn > 0; nn--)
- {
- uint8x8_t _src0 = vld1_u8(src0);
- uint8x8_t _src1 = vld1_u8(src1);
-
- uint8x8_t _src2 = vld1_u8(src0 + src_step);
- uint8x8_t _src3 = vld1_u8(src1 + src_step);
-
- uint8x8_t _src4 = vld1_u8(src0 + 2 * src_step);
- uint8x8_t _src5 = vld1_u8(src1 + 2 * src_step);
-
- uint8x8_t _src6 = vld1_u8(src0 + 3 * src_step);
- uint8x8_t _src7 = vld1_u8(src1 + 3 * src_step);
-
- uint8x8x2_t _src01t_r = vtrn_u8(_src1, _src0);
- uint8x8x2_t _src23t_r = vtrn_u8(_src3, _src2);
- uint8x8x2_t _src45t_r = vtrn_u8(_src5, _src4);
- uint8x8x2_t _src67t_r = vtrn_u8(_src7, _src6);
-
- uint16x4x2_t _src02tt_r = vtrn_u16(vreinterpret_u16_u8(_src23t_r.val[1]), vreinterpret_u16_u8(_src01t_r.val[1]));
- uint16x4x2_t _src13tt_r = vtrn_u16(vreinterpret_u16_u8(_src23t_r.val[0]), vreinterpret_u16_u8(_src01t_r.val[0]));
- uint16x4x2_t _src46tt_r = vtrn_u16(vreinterpret_u16_u8(_src67t_r.val[1]), vreinterpret_u16_u8(_src45t_r.val[1]));
- uint16x4x2_t _src57tt_r = vtrn_u16(vreinterpret_u16_u8(_src67t_r.val[0]), vreinterpret_u16_u8(_src45t_r.val[0]));
-
- uint32x2x2_t _src04ttt_r = vtrn_u32(vreinterpret_u32_u16(_src46tt_r.val[1]), vreinterpret_u32_u16(_src02tt_r.val[1]));
- uint32x2x2_t _src15ttt_r = vtrn_u32(vreinterpret_u32_u16(_src57tt_r.val[1]), vreinterpret_u32_u16(_src13tt_r.val[1]));
- uint32x2x2_t _src26ttt_r = vtrn_u32(vreinterpret_u32_u16(_src46tt_r.val[0]), vreinterpret_u32_u16(_src02tt_r.val[0]));
- uint32x2x2_t _src37ttt_r = vtrn_u32(vreinterpret_u32_u16(_src57tt_r.val[0]), vreinterpret_u32_u16(_src13tt_r.val[0]));
-
- uint8x8_t _dst0 = vreinterpret_u8_u32(_src04ttt_r.val[1]);
- uint8x8_t _dst1 = vreinterpret_u8_u32(_src15ttt_r.val[1]);
- uint8x8_t _dst2 = vreinterpret_u8_u32(_src26ttt_r.val[1]);
- uint8x8_t _dst3 = vreinterpret_u8_u32(_src37ttt_r.val[1]);
- uint8x8_t _dst4 = vreinterpret_u8_u32(_src04ttt_r.val[0]);
- uint8x8_t _dst5 = vreinterpret_u8_u32(_src15ttt_r.val[0]);
- uint8x8_t _dst6 = vreinterpret_u8_u32(_src26ttt_r.val[0]);
- uint8x8_t _dst7 = vreinterpret_u8_u32(_src37ttt_r.val[0]);
-
- vst1_u8(dst0, _dst7);
- vst1_u8(dst1, _dst6);
- vst1_u8(dst0 + dst_step, _dst5);
- vst1_u8(dst1 + dst_step, _dst4);
- vst1_u8(dst0 + 2 * dst_step, _dst3);
- vst1_u8(dst1 + 2 * dst_step, _dst2);
- vst1_u8(dst0 + 3 * dst_step, _dst1);
- vst1_u8(dst1 + 3 * dst_step, _dst0);
-
- src0 += 8;
- src1 += 8;
-
- dst0 += 4 * dst_step;
- dst1 += 4 * dst_step;
- }
- #else
- if (nn > 0)
- {
- asm volatile(
- "0: \n"
- "pld [%1, #64] \n"
- "vld1.u8 {d0}, [%1], %10 \n"
-
- "pld [%2, #64] \n"
- "vld1.u8 {d1}, [%2], %10 \n"
-
- "pld [%1, #64] \n"
- "vld1.u8 {d2}, [%1], %10 \n"
-
- "vtrn.u8 d1, d0 \n" // _src01t_r
-
- "pld [%2, #64] \n"
- "vld1.u8 {d3}, [%2], %10 \n"
-
- "pld [%1, #64] \n"
- "vld1.u8 {d4}, [%1], %10 \n"
-
- "vtrn.u8 d3, d2 \n" // _src23t_r
-
- "pld [%2, #64] \n"
- "vld1.u8 {d5}, [%2], %10 \n"
-
- "pld [%1, #64] \n"
- "vld1.u8 {d6}, [%1], %10 \n"
-
- "vtrn.u8 d5, d4 \n" // _src45t_r
-
- "pld [%2, #64] \n"
- "vld1.u8 {d7}, [%2], %10 \n"
-
- "vtrn.u8 d7, d6 \n" // _src67t_r
-
- "sub %1, %1, %10, lsl #2 \n" // restore src0
-
- "vtrn.u16 q1, q0 \n" // _src02tt_r _src13tt_r
-
- "sub %2, %2, %10, lsl #2 \n" // restore src1
-
- "vtrn.u16 q3, q2 \n" // _src46tt_r _src57tt_r
-
- "add %1, #8 \n" // src0 += 8
-
- "vtrn.u32 q3, q1 \n" // _src26ttt_r _src37ttt_r
-
- "add %2, #8 \n" // src1 += 8
-
- "vtrn.u32 q2, q0 \n" // _src04ttt_r _src15ttt_r
- "vst1.u8 {d6}, [%4], %11 \n"
- "vst1.u8 {d7}, [%3], %11 \n"
-
- "subs %0, #1 \n"
-
- "vst1.u8 {d4}, [%4], %11 \n"
- "vst1.u8 {d5}, [%3], %11 \n"
- "vst1.u8 {d2}, [%4], %11 \n"
- "vst1.u8 {d3}, [%3], %11 \n"
- "vst1.u8 {d0}, [%4], %11 \n"
- "vst1.u8 {d1}, [%3], %11 \n"
-
- "bne 0b \n"
- : "=r"(nn), // %0
- "=r"(src0), // %1
- "=r"(src1), // %2
- "=r"(dst0), // %3
- "=r"(dst1) // %4
- : "0"(nn),
- "1"(src0),
- "2"(src1),
- "3"(dst0),
- "4"(dst1),
- "r"(src_step), // %10
- "r"(dst_step) // %11
- : "cc", "memory", "q0", "q1", "q2", "q3");
- }
- #endif // __aarch64__
- for (; remain > 0; remain--)
- {
- dst0[0] = src1[0 + 3 * src_step];
- dst0[1] = src0[0 + 3 * src_step];
- dst0[2] = src1[0 + 2 * src_step];
- dst0[3] = src0[0 + 2 * src_step];
- dst0[4] = src1[0 + src_step];
- dst0[5] = src0[0 + src_step];
- dst0[6] = src1[0];
- dst0[7] = src0[0];
-
- src0 += 1;
- src1 += 1;
-
- dst0 += stride;
- }
-
- src0 += srcwgap + 7 * srcstride;
- }
- #endif // __ARM_NEON
- for (; y < srch; y++)
- {
- unsigned char* dst0 = dstend - y - 1;
-
- int x = 0;
- for (; x < srcw; x++)
- {
- *dst0 = *src0;
-
- src0 += 1;
- dst0 += stride;
- }
-
- src0 += srcwgap;
- }
- }
-
- static void kanna_rotate_6_c2(const unsigned char* src, int srcw, int srch, int srcstride, unsigned char* dst, int w, int /*h*/, int stride)
- {
- const int srcwgap = srcstride - srcw * 2;
-
- // point to the last dst pixel in row
- unsigned char* dstend = dst + w * 2;
-
- const unsigned char* src0 = src;
-
- int y = 0;
- #if __ARM_NEON
- for (; y + 7 < srch; y += 8)
- {
- const unsigned char* src1 = src0 + srcstride;
-
- unsigned char* dst0 = dstend - y * 2 - 8 * 2;
- unsigned char* dst1 = dstend - y * 2 - 8 * 2 + stride;
-
- int src_step = 2 * srcstride;
- int dst_step = 2 * stride;
-
- int nn = srcw >> 3;
- int remain = srcw - (nn << 3);
-
- #if !NCNN_GNU_INLINE_ASM || __aarch64__
- for (; nn > 0; nn--)
- {
- uint8x8x2_t _src0 = vld2_u8(src0);
- uint8x8x2_t _src1 = vld2_u8(src1);
-
- uint8x8x2_t _src2 = vld2_u8(src0 + src_step);
- uint8x8x2_t _src3 = vld2_u8(src1 + src_step);
-
- uint8x8x2_t _src4 = vld2_u8(src0 + 2 * src_step);
- uint8x8x2_t _src5 = vld2_u8(src1 + 2 * src_step);
-
- uint8x8x2_t _src6 = vld2_u8(src0 + 3 * src_step);
- uint8x8x2_t _src7 = vld2_u8(src1 + 3 * src_step);
-
- uint8x8x2_t _src01t_r = vtrn_u8(_src1.val[0], _src0.val[0]);
- uint8x8x2_t _src23t_r = vtrn_u8(_src3.val[0], _src2.val[0]);
- uint8x8x2_t _src45t_r = vtrn_u8(_src5.val[0], _src4.val[0]);
- uint8x8x2_t _src67t_r = vtrn_u8(_src7.val[0], _src6.val[0]);
-
- uint8x8x2_t _src01t_g = vtrn_u8(_src1.val[1], _src0.val[1]);
- uint8x8x2_t _src23t_g = vtrn_u8(_src3.val[1], _src2.val[1]);
- uint8x8x2_t _src45t_g = vtrn_u8(_src5.val[1], _src4.val[1]);
- uint8x8x2_t _src67t_g = vtrn_u8(_src7.val[1], _src6.val[1]);
-
- uint16x4x2_t _src02tt_r = vtrn_u16(vreinterpret_u16_u8(_src23t_r.val[1]), vreinterpret_u16_u8(_src01t_r.val[1]));
- uint16x4x2_t _src13tt_r = vtrn_u16(vreinterpret_u16_u8(_src23t_r.val[0]), vreinterpret_u16_u8(_src01t_r.val[0]));
- uint16x4x2_t _src46tt_r = vtrn_u16(vreinterpret_u16_u8(_src67t_r.val[1]), vreinterpret_u16_u8(_src45t_r.val[1]));
- uint16x4x2_t _src57tt_r = vtrn_u16(vreinterpret_u16_u8(_src67t_r.val[0]), vreinterpret_u16_u8(_src45t_r.val[0]));
-
- uint16x4x2_t _src02tt_g = vtrn_u16(vreinterpret_u16_u8(_src23t_g.val[1]), vreinterpret_u16_u8(_src01t_g.val[1]));
- uint16x4x2_t _src13tt_g = vtrn_u16(vreinterpret_u16_u8(_src23t_g.val[0]), vreinterpret_u16_u8(_src01t_g.val[0]));
- uint16x4x2_t _src46tt_g = vtrn_u16(vreinterpret_u16_u8(_src67t_g.val[1]), vreinterpret_u16_u8(_src45t_g.val[1]));
- uint16x4x2_t _src57tt_g = vtrn_u16(vreinterpret_u16_u8(_src67t_g.val[0]), vreinterpret_u16_u8(_src45t_g.val[0]));
-
- uint32x2x2_t _src04ttt_r = vtrn_u32(vreinterpret_u32_u16(_src46tt_r.val[1]), vreinterpret_u32_u16(_src02tt_r.val[1]));
- uint32x2x2_t _src15ttt_r = vtrn_u32(vreinterpret_u32_u16(_src57tt_r.val[1]), vreinterpret_u32_u16(_src13tt_r.val[1]));
- uint32x2x2_t _src26ttt_r = vtrn_u32(vreinterpret_u32_u16(_src46tt_r.val[0]), vreinterpret_u32_u16(_src02tt_r.val[0]));
- uint32x2x2_t _src37ttt_r = vtrn_u32(vreinterpret_u32_u16(_src57tt_r.val[0]), vreinterpret_u32_u16(_src13tt_r.val[0]));
-
- uint32x2x2_t _src04ttt_g = vtrn_u32(vreinterpret_u32_u16(_src46tt_g.val[1]), vreinterpret_u32_u16(_src02tt_g.val[1]));
- uint32x2x2_t _src15ttt_g = vtrn_u32(vreinterpret_u32_u16(_src57tt_g.val[1]), vreinterpret_u32_u16(_src13tt_g.val[1]));
- uint32x2x2_t _src26ttt_g = vtrn_u32(vreinterpret_u32_u16(_src46tt_g.val[0]), vreinterpret_u32_u16(_src02tt_g.val[0]));
- uint32x2x2_t _src37ttt_g = vtrn_u32(vreinterpret_u32_u16(_src57tt_g.val[0]), vreinterpret_u32_u16(_src13tt_g.val[0]));
-
- uint8x8x2_t _dst0;
- uint8x8x2_t _dst1;
- uint8x8x2_t _dst2;
- uint8x8x2_t _dst3;
- uint8x8x2_t _dst4;
- uint8x8x2_t _dst5;
- uint8x8x2_t _dst6;
- uint8x8x2_t _dst7;
-
- _dst0.val[0] = vreinterpret_u8_u32(_src04ttt_r.val[1]);
- _dst1.val[0] = vreinterpret_u8_u32(_src15ttt_r.val[1]);
- _dst2.val[0] = vreinterpret_u8_u32(_src26ttt_r.val[1]);
- _dst3.val[0] = vreinterpret_u8_u32(_src37ttt_r.val[1]);
- _dst4.val[0] = vreinterpret_u8_u32(_src04ttt_r.val[0]);
- _dst5.val[0] = vreinterpret_u8_u32(_src15ttt_r.val[0]);
- _dst6.val[0] = vreinterpret_u8_u32(_src26ttt_r.val[0]);
- _dst7.val[0] = vreinterpret_u8_u32(_src37ttt_r.val[0]);
-
- _dst0.val[1] = vreinterpret_u8_u32(_src04ttt_g.val[1]);
- _dst1.val[1] = vreinterpret_u8_u32(_src15ttt_g.val[1]);
- _dst2.val[1] = vreinterpret_u8_u32(_src26ttt_g.val[1]);
- _dst3.val[1] = vreinterpret_u8_u32(_src37ttt_g.val[1]);
- _dst4.val[1] = vreinterpret_u8_u32(_src04ttt_g.val[0]);
- _dst5.val[1] = vreinterpret_u8_u32(_src15ttt_g.val[0]);
- _dst6.val[1] = vreinterpret_u8_u32(_src26ttt_g.val[0]);
- _dst7.val[1] = vreinterpret_u8_u32(_src37ttt_g.val[0]);
-
- vst2_u8(dst0, _dst7);
- vst2_u8(dst1, _dst6);
- vst2_u8(dst0 + dst_step, _dst5);
- vst2_u8(dst1 + dst_step, _dst4);
- vst2_u8(dst0 + 2 * dst_step, _dst3);
- vst2_u8(dst1 + 2 * dst_step, _dst2);
- vst2_u8(dst0 + 3 * dst_step, _dst1);
- vst2_u8(dst1 + 3 * dst_step, _dst0);
-
- src0 += 2 * 8;
- src1 += 2 * 8;
-
- dst0 += 4 * dst_step;
- dst1 += 4 * dst_step;
- }
- #else
- if (nn > 0)
- {
- asm volatile(
- "0: \n"
- "pld [%1, #128] \n"
- "vld2.u8 {d0-d1}, [%1], %10 \n"
-
- "pld [%2, #128] \n"
- "vld2.u8 {d2-d3}, [%2], %10 \n"
-
- "pld [%1, #128] \n"
- "vld2.u8 {d4-d5}, [%1], %10 \n"
-
- "vtrn.u8 q1, q0 \n" // _src01t_r
-
- "pld [%2, #128] \n"
- "vld2.u8 {d6-d7}, [%2], %10 \n"
-
- "pld [%1, #128] \n"
- "vld2.u8 {d16-d17}, [%1], %10\n"
-
- "vtrn.u8 q3, q2 \n" // _src23t_r
-
- "pld [%2, #128] \n"
- "vld2.u8 {d18-d19}, [%2], %10\n"
-
- "pld [%1, #128] \n"
- "vld2.u8 {d20-d21}, [%1], %10\n"
-
- "vtrn.u8 q9, q8 \n" // _src45t_r
-
- "pld [%2, #128] \n"
- "vld2.u8 {d22-d23}, [%2], %10\n"
-
- "vtrn.u8 q11, q10 \n" // _src67t_r
-
- "sub %1, %1, %10, lsl #2 \n" // restore src0
-
- "vtrn.u16 q2, q0 \n" // _src02tt_r
-
- "sub %2, %2, %10, lsl #2 \n" // restore src1
-
- "vtrn.u16 q3, q1 \n" // _src13tt_r
-
- "add %1, #16 \n" // src0 += 16
-
- "vtrn.u16 q10, q8 \n" // _src46tt_r
-
- "add %2, #16 \n" // src1 += 16
-
- "vtrn.u16 q11, q9 \n" // _src57tt_r
-
- "vtrn.u32 q10, q2 \n" // _src26ttt_r
-
- "vtrn.u32 q11, q3 \n" // _src37ttt_r
- "vst2.u8 {d20-d21}, [%4], %11\n"
-
- "vtrn.u32 q8, q0 \n" // _src04ttt_r
- "vst2.u8 {d22-d23}, [%3], %11\n"
-
- "vtrn.u32 q9, q1 \n" // _src15ttt_r
- "vst2.u8 {d16-d17}, [%4], %11\n"
-
- "subs %0, #1 \n"
-
- "vst2.u8 {d18-d19}, [%3], %11\n"
- "vst2.u8 {d4-d5}, [%4], %11 \n"
- "vst2.u8 {d6-d7}, [%3], %11 \n"
- "vst2.u8 {d0-d1}, [%4], %11 \n"
- "vst2.u8 {d2-d3}, [%3], %11 \n"
-
- "bne 0b \n"
- : "=r"(nn), // %0
- "=r"(src0), // %1
- "=r"(src1), // %2
- "=r"(dst0), // %3
- "=r"(dst1) // %4
- : "0"(nn),
- "1"(src0),
- "2"(src1),
- "3"(dst0),
- "4"(dst1),
- "r"(src_step), // %10
- "r"(dst_step) // %11
- : "cc", "memory", "q0", "q1", "q2", "q3", "q8", "q9", "q10", "q11");
- }
- #endif // __aarch64__
- for (; remain > 0; remain--)
- {
- dst0[0] = src1[0 + 3 * src_step];
- dst0[1] = src1[1 + 3 * src_step];
- dst0[2] = src0[0 + 3 * src_step];
- dst0[3] = src0[1 + 3 * src_step];
- dst0[4] = src1[0 + 2 * src_step];
- dst0[5] = src1[1 + 2 * src_step];
- dst0[6] = src0[0 + 2 * src_step];
- dst0[7] = src0[1 + 2 * src_step];
- dst0[8] = src1[0 + src_step];
- dst0[9] = src1[1 + src_step];
- dst0[10] = src0[0 + src_step];
- dst0[11] = src0[1 + src_step];
- dst0[12] = src1[0];
- dst0[13] = src1[1];
- dst0[14] = src0[0];
- dst0[15] = src0[1];
-
- src0 += 2;
- src1 += 2;
-
- dst0 += stride;
- }
-
- src0 += srcwgap + 7 * srcstride;
- }
- #endif // __ARM_NEON
- for (; y < srch; y++)
- {
- unsigned char* dst0 = dstend - y * 2 - 2;
-
- int x = 0;
- for (; x < srcw; x++)
- {
- dst0[0] = src0[0];
- dst0[1] = src0[1];
-
- src0 += 2;
- dst0 += stride;
- }
-
- src0 += srcwgap;
- }
- }
-
- static void kanna_rotate_6_c3(const unsigned char* src, int srcw, int srch, int srcstride, unsigned char* dst, int w, int /*h*/, int stride)
- {
- const int srcwgap = srcstride - srcw * 3;
-
- // point to the last dst pixel in row
- unsigned char* dstend = dst + w * 3;
-
- const unsigned char* src0 = src;
-
- int y = 0;
- #if __ARM_NEON
- for (; y + 7 < srch; y += 8)
- {
- const unsigned char* src1 = src0 + srcstride;
-
- unsigned char* dst0 = dstend - y * 3 - 8 * 3;
- unsigned char* dst1 = dstend - y * 3 - 8 * 3 + stride;
-
- int src_step = 2 * srcstride;
- int dst_step = 2 * stride;
-
- int nn = srcw >> 3;
- int remain = srcw - (nn << 3);
-
- #if !NCNN_GNU_INLINE_ASM || __aarch64__
- for (; nn > 0; nn--)
- {
- uint8x8x3_t _src0 = vld3_u8(src0);
- uint8x8x3_t _src1 = vld3_u8(src1);
-
- uint8x8x3_t _src2 = vld3_u8(src0 + src_step);
- uint8x8x3_t _src3 = vld3_u8(src1 + src_step);
-
- uint8x8x3_t _src4 = vld3_u8(src0 + 2 * src_step);
- uint8x8x3_t _src5 = vld3_u8(src1 + 2 * src_step);
-
- uint8x8x3_t _src6 = vld3_u8(src0 + 3 * src_step);
- uint8x8x3_t _src7 = vld3_u8(src1 + 3 * src_step);
-
- uint8x8x2_t _src01t_r = vtrn_u8(_src1.val[0], _src0.val[0]);
- uint8x8x2_t _src23t_r = vtrn_u8(_src3.val[0], _src2.val[0]);
- uint8x8x2_t _src45t_r = vtrn_u8(_src5.val[0], _src4.val[0]);
- uint8x8x2_t _src67t_r = vtrn_u8(_src7.val[0], _src6.val[0]);
-
- uint8x8x2_t _src01t_g = vtrn_u8(_src1.val[1], _src0.val[1]);
- uint8x8x2_t _src23t_g = vtrn_u8(_src3.val[1], _src2.val[1]);
- uint8x8x2_t _src45t_g = vtrn_u8(_src5.val[1], _src4.val[1]);
- uint8x8x2_t _src67t_g = vtrn_u8(_src7.val[1], _src6.val[1]);
-
- uint8x8x2_t _src01t_b = vtrn_u8(_src1.val[2], _src0.val[2]);
- uint8x8x2_t _src23t_b = vtrn_u8(_src3.val[2], _src2.val[2]);
- uint8x8x2_t _src45t_b = vtrn_u8(_src5.val[2], _src4.val[2]);
- uint8x8x2_t _src67t_b = vtrn_u8(_src7.val[2], _src6.val[2]);
-
- uint16x4x2_t _src02tt_r = vtrn_u16(vreinterpret_u16_u8(_src23t_r.val[1]), vreinterpret_u16_u8(_src01t_r.val[1]));
- uint16x4x2_t _src13tt_r = vtrn_u16(vreinterpret_u16_u8(_src23t_r.val[0]), vreinterpret_u16_u8(_src01t_r.val[0]));
- uint16x4x2_t _src46tt_r = vtrn_u16(vreinterpret_u16_u8(_src67t_r.val[1]), vreinterpret_u16_u8(_src45t_r.val[1]));
- uint16x4x2_t _src57tt_r = vtrn_u16(vreinterpret_u16_u8(_src67t_r.val[0]), vreinterpret_u16_u8(_src45t_r.val[0]));
-
- uint16x4x2_t _src02tt_g = vtrn_u16(vreinterpret_u16_u8(_src23t_g.val[1]), vreinterpret_u16_u8(_src01t_g.val[1]));
- uint16x4x2_t _src13tt_g = vtrn_u16(vreinterpret_u16_u8(_src23t_g.val[0]), vreinterpret_u16_u8(_src01t_g.val[0]));
- uint16x4x2_t _src46tt_g = vtrn_u16(vreinterpret_u16_u8(_src67t_g.val[1]), vreinterpret_u16_u8(_src45t_g.val[1]));
- uint16x4x2_t _src57tt_g = vtrn_u16(vreinterpret_u16_u8(_src67t_g.val[0]), vreinterpret_u16_u8(_src45t_g.val[0]));
-
- uint16x4x2_t _src02tt_b = vtrn_u16(vreinterpret_u16_u8(_src23t_b.val[1]), vreinterpret_u16_u8(_src01t_b.val[1]));
- uint16x4x2_t _src13tt_b = vtrn_u16(vreinterpret_u16_u8(_src23t_b.val[0]), vreinterpret_u16_u8(_src01t_b.val[0]));
- uint16x4x2_t _src46tt_b = vtrn_u16(vreinterpret_u16_u8(_src67t_b.val[1]), vreinterpret_u16_u8(_src45t_b.val[1]));
- uint16x4x2_t _src57tt_b = vtrn_u16(vreinterpret_u16_u8(_src67t_b.val[0]), vreinterpret_u16_u8(_src45t_b.val[0]));
-
- uint32x2x2_t _src04ttt_r = vtrn_u32(vreinterpret_u32_u16(_src46tt_r.val[1]), vreinterpret_u32_u16(_src02tt_r.val[1]));
- uint32x2x2_t _src15ttt_r = vtrn_u32(vreinterpret_u32_u16(_src57tt_r.val[1]), vreinterpret_u32_u16(_src13tt_r.val[1]));
- uint32x2x2_t _src26ttt_r = vtrn_u32(vreinterpret_u32_u16(_src46tt_r.val[0]), vreinterpret_u32_u16(_src02tt_r.val[0]));
- uint32x2x2_t _src37ttt_r = vtrn_u32(vreinterpret_u32_u16(_src57tt_r.val[0]), vreinterpret_u32_u16(_src13tt_r.val[0]));
-
- uint32x2x2_t _src04ttt_g = vtrn_u32(vreinterpret_u32_u16(_src46tt_g.val[1]), vreinterpret_u32_u16(_src02tt_g.val[1]));
- uint32x2x2_t _src15ttt_g = vtrn_u32(vreinterpret_u32_u16(_src57tt_g.val[1]), vreinterpret_u32_u16(_src13tt_g.val[1]));
- uint32x2x2_t _src26ttt_g = vtrn_u32(vreinterpret_u32_u16(_src46tt_g.val[0]), vreinterpret_u32_u16(_src02tt_g.val[0]));
- uint32x2x2_t _src37ttt_g = vtrn_u32(vreinterpret_u32_u16(_src57tt_g.val[0]), vreinterpret_u32_u16(_src13tt_g.val[0]));
-
- uint32x2x2_t _src04ttt_b = vtrn_u32(vreinterpret_u32_u16(_src46tt_b.val[1]), vreinterpret_u32_u16(_src02tt_b.val[1]));
- uint32x2x2_t _src15ttt_b = vtrn_u32(vreinterpret_u32_u16(_src57tt_b.val[1]), vreinterpret_u32_u16(_src13tt_b.val[1]));
- uint32x2x2_t _src26ttt_b = vtrn_u32(vreinterpret_u32_u16(_src46tt_b.val[0]), vreinterpret_u32_u16(_src02tt_b.val[0]));
- uint32x2x2_t _src37ttt_b = vtrn_u32(vreinterpret_u32_u16(_src57tt_b.val[0]), vreinterpret_u32_u16(_src13tt_b.val[0]));
-
- uint8x8x3_t _dst0;
- uint8x8x3_t _dst1;
- uint8x8x3_t _dst2;
- uint8x8x3_t _dst3;
- uint8x8x3_t _dst4;
- uint8x8x3_t _dst5;
- uint8x8x3_t _dst6;
- uint8x8x3_t _dst7;
-
- _dst0.val[0] = vreinterpret_u8_u32(_src04ttt_r.val[1]);
- _dst1.val[0] = vreinterpret_u8_u32(_src15ttt_r.val[1]);
- _dst2.val[0] = vreinterpret_u8_u32(_src26ttt_r.val[1]);
- _dst3.val[0] = vreinterpret_u8_u32(_src37ttt_r.val[1]);
- _dst4.val[0] = vreinterpret_u8_u32(_src04ttt_r.val[0]);
- _dst5.val[0] = vreinterpret_u8_u32(_src15ttt_r.val[0]);
- _dst6.val[0] = vreinterpret_u8_u32(_src26ttt_r.val[0]);
- _dst7.val[0] = vreinterpret_u8_u32(_src37ttt_r.val[0]);
-
- _dst0.val[1] = vreinterpret_u8_u32(_src04ttt_g.val[1]);
- _dst1.val[1] = vreinterpret_u8_u32(_src15ttt_g.val[1]);
- _dst2.val[1] = vreinterpret_u8_u32(_src26ttt_g.val[1]);
- _dst3.val[1] = vreinterpret_u8_u32(_src37ttt_g.val[1]);
- _dst4.val[1] = vreinterpret_u8_u32(_src04ttt_g.val[0]);
- _dst5.val[1] = vreinterpret_u8_u32(_src15ttt_g.val[0]);
- _dst6.val[1] = vreinterpret_u8_u32(_src26ttt_g.val[0]);
- _dst7.val[1] = vreinterpret_u8_u32(_src37ttt_g.val[0]);
-
- _dst0.val[2] = vreinterpret_u8_u32(_src04ttt_b.val[1]);
- _dst1.val[2] = vreinterpret_u8_u32(_src15ttt_b.val[1]);
- _dst2.val[2] = vreinterpret_u8_u32(_src26ttt_b.val[1]);
- _dst3.val[2] = vreinterpret_u8_u32(_src37ttt_b.val[1]);
- _dst4.val[2] = vreinterpret_u8_u32(_src04ttt_b.val[0]);
- _dst5.val[2] = vreinterpret_u8_u32(_src15ttt_b.val[0]);
- _dst6.val[2] = vreinterpret_u8_u32(_src26ttt_b.val[0]);
- _dst7.val[2] = vreinterpret_u8_u32(_src37ttt_b.val[0]);
-
- vst3_u8(dst0, _dst7);
- vst3_u8(dst1, _dst6);
- vst3_u8(dst0 + dst_step, _dst5);
- vst3_u8(dst1 + dst_step, _dst4);
- vst3_u8(dst0 + 2 * dst_step, _dst3);
- vst3_u8(dst1 + 2 * dst_step, _dst2);
- vst3_u8(dst0 + 3 * dst_step, _dst1);
- vst3_u8(dst1 + 3 * dst_step, _dst0);
-
- src0 += 3 * 8;
- src1 += 3 * 8;
-
- dst0 += 4 * dst_step;
- dst1 += 4 * dst_step;
- }
- #else
- if (nn > 0)
- {
- asm volatile(
- "0: \n"
- "pld [%1, #192] \n"
- "vld3.u8 {d0-d2}, [%1], %10 \n"
-
- "pld [%2, #192] \n"
- "vld3.u8 {d4-d6}, [%2], %10 \n"
-
- "pld [%1, #192] \n"
- "vld3.u8 {d8-d10}, [%1], %10 \n"
-
- "vtrn.u8 q2, q0 \n" // _src01t_r
- "vtrn.u8 d6, d2 \n"
-
- "pld [%2, #192] \n"
- "vld3.u8 {d12-d14}, [%2], %10\n"
-
- "pld [%1, #192] \n"
- "vld3.u8 {d16-d18}, [%1], %10\n"
-
- "vtrn.u8 q6, q4 \n" // _src23t_r
- "vtrn.u8 d14, d10 \n"
-
- "pld [%2, #192] \n"
- "vld3.u8 {d20-d22}, [%2], %10\n"
-
- "pld [%1, #192] \n"
- "vld3.u8 {d24-d26}, [%1], %10\n"
-
- "vtrn.u8 q10, q8 \n" // _src45t_r
- "vtrn.u8 d22, d18 \n"
-
- "pld [%2, #192] \n"
- "vld3.u8 {d28-d30}, [%2], %10\n"
-
- "vtrn.u8 q14, q12 \n" // _src67t_r
- "vtrn.u8 d30, d26 \n"
-
- "sub %1, %1, %10, lsl #2 \n" // restore src0
-
- "vtrn.u16 q4, q0 \n" // _src02tt_r
- "vtrn.u16 d10, d2 \n"
-
- "sub %2, %2, %10, lsl #2 \n" // restore src1
-
- "vtrn.u16 q6, q2 \n" // _src13tt_r
- "vtrn.u16 d14, d6 \n"
-
- "add %1, #24 \n" // src0 += 24
-
- "vtrn.u16 q12, q8 \n" // _src46tt_r
- "vtrn.u16 d26, d18 \n"
-
- "add %2, #24 \n" // src1 += 24
-
- "vtrn.u16 q14, q10 \n" // _src57tt_r
- "vtrn.u16 d30, d22 \n"
-
- "vtrn.u32 q12, q4 \n" // _src26ttt_r
- "vtrn.u32 d26, d10 \n"
-
- "vtrn.u32 q14, q6 \n" // _src37ttt_r
- "vst3.u8 {d24-d26}, [%4], %11\n"
- "vtrn.u32 d30, d14 \n"
-
- "vtrn.u32 q8, q0 \n" // _src04ttt_r
- "vst3.u8 {d28-d30}, [%3], %11\n"
- "vtrn.u32 d18, d2 \n"
-
- "vtrn.u32 q10, q2 \n" // _src15ttt_r
- "vst3.u8 {d16-d18}, [%4], %11\n"
- "vtrn.u32 d22, d6 \n"
-
- "subs %0, #1 \n"
-
- "vst3.u8 {d20-d22}, [%3], %11\n"
- "vst3.u8 {d8-d10}, [%4], %11 \n"
- "vst3.u8 {d12-d14}, [%3], %11\n"
- "vst3.u8 {d0-d2}, [%4], %11 \n"
- "vst3.u8 {d4-d6}, [%3], %11 \n"
-
- "bne 0b \n"
- : "=r"(nn), // %0
- "=r"(src0), // %1
- "=r"(src1), // %2
- "=r"(dst0), // %3
- "=r"(dst1) // %4
- : "0"(nn),
- "1"(src0),
- "2"(src1),
- "3"(dst0),
- "4"(dst1),
- "r"(src_step), // %10
- "r"(dst_step) // %11
- : "cc", "memory", "q0", "q1", "q2", "q3", "q4", "q5", "q6", "q7", "q8", "q9", "q10", "q11", "q12", "q13", "q14", "q15");
- }
- #endif // __aarch64__
- for (; remain > 0; remain--)
- {
- dst0[0] = src1[0 + 3 * src_step];
- dst0[1] = src1[1 + 3 * src_step];
- dst0[2] = src1[2 + 3 * src_step];
- dst0[3] = src0[0 + 3 * src_step];
- dst0[4] = src0[1 + 3 * src_step];
- dst0[5] = src0[2 + 3 * src_step];
- dst0[6] = src1[0 + 2 * src_step];
- dst0[7] = src1[1 + 2 * src_step];
- dst0[8] = src1[2 + 2 * src_step];
- dst0[9] = src0[0 + 2 * src_step];
- dst0[10] = src0[1 + 2 * src_step];
- dst0[11] = src0[2 + 2 * src_step];
- dst0[12] = src1[0 + src_step];
- dst0[13] = src1[1 + src_step];
- dst0[14] = src1[2 + src_step];
- dst0[15] = src0[0 + src_step];
- dst0[16] = src0[1 + src_step];
- dst0[17] = src0[2 + src_step];
- dst0[18] = src1[0];
- dst0[19] = src1[1];
- dst0[20] = src1[2];
- dst0[21] = src0[0];
- dst0[22] = src0[1];
- dst0[23] = src0[2];
-
- src0 += 3;
- src1 += 3;
-
- dst0 += stride;
- }
-
- src0 += srcwgap + 7 * srcstride;
- }
- #endif // __ARM_NEON
- for (; y < srch; y++)
- {
- unsigned char* dst0 = dstend - y * 3 - 3;
-
- int x = 0;
- for (; x < srcw; x++)
- {
- dst0[0] = src0[0];
- dst0[1] = src0[1];
- dst0[2] = src0[2];
-
- src0 += 3;
- dst0 += stride;
- }
-
- src0 += srcwgap;
- }
- }
-
- static void kanna_rotate_6_c4(const unsigned char* src, int srcw, int srch, int srcstride, unsigned char* dst, int w, int /*h*/, int stride)
- {
- const int srcwgap = srcstride - srcw * 4;
-
- // point to the last dst pixel in row
- unsigned char* dstend = dst + w * 4;
-
- const unsigned char* src0 = src;
-
- int y = 0;
- #if __ARM_NEON
- for (; y + 7 < srch; y += 8)
- {
- const unsigned char* src1 = src0 + srcstride;
-
- unsigned char* dst0 = dstend - y * 4 - 8 * 4;
- unsigned char* dst1 = dstend - y * 4 - 8 * 4 + stride;
-
- int src_step = 2 * srcstride;
- int dst_step = 2 * stride;
-
- int nn = srcw >> 3;
- int remain = srcw - (nn << 3);
-
- #if !NCNN_GNU_INLINE_ASM || __aarch64__
- for (; nn > 0; nn--)
- {
- uint8x8x4_t _src0 = vld4_u8(src0);
- uint8x8x4_t _src1 = vld4_u8(src1);
-
- uint8x8x4_t _src2 = vld4_u8(src0 + src_step);
- uint8x8x4_t _src3 = vld4_u8(src1 + src_step);
-
- uint8x8x4_t _src4 = vld4_u8(src0 + 2 * src_step);
- uint8x8x4_t _src5 = vld4_u8(src1 + 2 * src_step);
-
- uint8x8x4_t _src6 = vld4_u8(src0 + 3 * src_step);
- uint8x8x4_t _src7 = vld4_u8(src1 + 3 * src_step);
-
- uint8x8x2_t _src01t_r = vtrn_u8(_src1.val[0], _src0.val[0]);
- uint8x8x2_t _src23t_r = vtrn_u8(_src3.val[0], _src2.val[0]);
- uint8x8x2_t _src45t_r = vtrn_u8(_src5.val[0], _src4.val[0]);
- uint8x8x2_t _src67t_r = vtrn_u8(_src7.val[0], _src6.val[0]);
-
- uint8x8x2_t _src01t_g = vtrn_u8(_src1.val[1], _src0.val[1]);
- uint8x8x2_t _src23t_g = vtrn_u8(_src3.val[1], _src2.val[1]);
- uint8x8x2_t _src45t_g = vtrn_u8(_src5.val[1], _src4.val[1]);
- uint8x8x2_t _src67t_g = vtrn_u8(_src7.val[1], _src6.val[1]);
-
- uint8x8x2_t _src01t_b = vtrn_u8(_src1.val[2], _src0.val[2]);
- uint8x8x2_t _src23t_b = vtrn_u8(_src3.val[2], _src2.val[2]);
- uint8x8x2_t _src45t_b = vtrn_u8(_src5.val[2], _src4.val[2]);
- uint8x8x2_t _src67t_b = vtrn_u8(_src7.val[2], _src6.val[2]);
-
- uint8x8x2_t _src01t_a = vtrn_u8(_src1.val[3], _src0.val[3]);
- uint8x8x2_t _src23t_a = vtrn_u8(_src3.val[3], _src2.val[3]);
- uint8x8x2_t _src45t_a = vtrn_u8(_src5.val[3], _src4.val[3]);
- uint8x8x2_t _src67t_a = vtrn_u8(_src7.val[3], _src6.val[3]);
-
- uint16x4x2_t _src02tt_r = vtrn_u16(vreinterpret_u16_u8(_src23t_r.val[1]), vreinterpret_u16_u8(_src01t_r.val[1]));
- uint16x4x2_t _src13tt_r = vtrn_u16(vreinterpret_u16_u8(_src23t_r.val[0]), vreinterpret_u16_u8(_src01t_r.val[0]));
- uint16x4x2_t _src46tt_r = vtrn_u16(vreinterpret_u16_u8(_src67t_r.val[1]), vreinterpret_u16_u8(_src45t_r.val[1]));
- uint16x4x2_t _src57tt_r = vtrn_u16(vreinterpret_u16_u8(_src67t_r.val[0]), vreinterpret_u16_u8(_src45t_r.val[0]));
-
- uint16x4x2_t _src02tt_g = vtrn_u16(vreinterpret_u16_u8(_src23t_g.val[1]), vreinterpret_u16_u8(_src01t_g.val[1]));
- uint16x4x2_t _src13tt_g = vtrn_u16(vreinterpret_u16_u8(_src23t_g.val[0]), vreinterpret_u16_u8(_src01t_g.val[0]));
- uint16x4x2_t _src46tt_g = vtrn_u16(vreinterpret_u16_u8(_src67t_g.val[1]), vreinterpret_u16_u8(_src45t_g.val[1]));
- uint16x4x2_t _src57tt_g = vtrn_u16(vreinterpret_u16_u8(_src67t_g.val[0]), vreinterpret_u16_u8(_src45t_g.val[0]));
-
- uint16x4x2_t _src02tt_b = vtrn_u16(vreinterpret_u16_u8(_src23t_b.val[1]), vreinterpret_u16_u8(_src01t_b.val[1]));
- uint16x4x2_t _src13tt_b = vtrn_u16(vreinterpret_u16_u8(_src23t_b.val[0]), vreinterpret_u16_u8(_src01t_b.val[0]));
- uint16x4x2_t _src46tt_b = vtrn_u16(vreinterpret_u16_u8(_src67t_b.val[1]), vreinterpret_u16_u8(_src45t_b.val[1]));
- uint16x4x2_t _src57tt_b = vtrn_u16(vreinterpret_u16_u8(_src67t_b.val[0]), vreinterpret_u16_u8(_src45t_b.val[0]));
-
- uint16x4x2_t _src02tt_a = vtrn_u16(vreinterpret_u16_u8(_src23t_a.val[1]), vreinterpret_u16_u8(_src01t_a.val[1]));
- uint16x4x2_t _src13tt_a = vtrn_u16(vreinterpret_u16_u8(_src23t_a.val[0]), vreinterpret_u16_u8(_src01t_a.val[0]));
- uint16x4x2_t _src46tt_a = vtrn_u16(vreinterpret_u16_u8(_src67t_a.val[1]), vreinterpret_u16_u8(_src45t_a.val[1]));
- uint16x4x2_t _src57tt_a = vtrn_u16(vreinterpret_u16_u8(_src67t_a.val[0]), vreinterpret_u16_u8(_src45t_a.val[0]));
-
- uint32x2x2_t _src04ttt_r = vtrn_u32(vreinterpret_u32_u16(_src46tt_r.val[1]), vreinterpret_u32_u16(_src02tt_r.val[1]));
- uint32x2x2_t _src15ttt_r = vtrn_u32(vreinterpret_u32_u16(_src57tt_r.val[1]), vreinterpret_u32_u16(_src13tt_r.val[1]));
- uint32x2x2_t _src26ttt_r = vtrn_u32(vreinterpret_u32_u16(_src46tt_r.val[0]), vreinterpret_u32_u16(_src02tt_r.val[0]));
- uint32x2x2_t _src37ttt_r = vtrn_u32(vreinterpret_u32_u16(_src57tt_r.val[0]), vreinterpret_u32_u16(_src13tt_r.val[0]));
-
- uint32x2x2_t _src04ttt_g = vtrn_u32(vreinterpret_u32_u16(_src46tt_g.val[1]), vreinterpret_u32_u16(_src02tt_g.val[1]));
- uint32x2x2_t _src15ttt_g = vtrn_u32(vreinterpret_u32_u16(_src57tt_g.val[1]), vreinterpret_u32_u16(_src13tt_g.val[1]));
- uint32x2x2_t _src26ttt_g = vtrn_u32(vreinterpret_u32_u16(_src46tt_g.val[0]), vreinterpret_u32_u16(_src02tt_g.val[0]));
- uint32x2x2_t _src37ttt_g = vtrn_u32(vreinterpret_u32_u16(_src57tt_g.val[0]), vreinterpret_u32_u16(_src13tt_g.val[0]));
-
- uint32x2x2_t _src04ttt_b = vtrn_u32(vreinterpret_u32_u16(_src46tt_b.val[1]), vreinterpret_u32_u16(_src02tt_b.val[1]));
- uint32x2x2_t _src15ttt_b = vtrn_u32(vreinterpret_u32_u16(_src57tt_b.val[1]), vreinterpret_u32_u16(_src13tt_b.val[1]));
- uint32x2x2_t _src26ttt_b = vtrn_u32(vreinterpret_u32_u16(_src46tt_b.val[0]), vreinterpret_u32_u16(_src02tt_b.val[0]));
- uint32x2x2_t _src37ttt_b = vtrn_u32(vreinterpret_u32_u16(_src57tt_b.val[0]), vreinterpret_u32_u16(_src13tt_b.val[0]));
-
- uint32x2x2_t _src04ttt_a = vtrn_u32(vreinterpret_u32_u16(_src46tt_a.val[1]), vreinterpret_u32_u16(_src02tt_a.val[1]));
- uint32x2x2_t _src15ttt_a = vtrn_u32(vreinterpret_u32_u16(_src57tt_a.val[1]), vreinterpret_u32_u16(_src13tt_a.val[1]));
- uint32x2x2_t _src26ttt_a = vtrn_u32(vreinterpret_u32_u16(_src46tt_a.val[0]), vreinterpret_u32_u16(_src02tt_a.val[0]));
- uint32x2x2_t _src37ttt_a = vtrn_u32(vreinterpret_u32_u16(_src57tt_a.val[0]), vreinterpret_u32_u16(_src13tt_a.val[0]));
-
- uint8x8x4_t _dst0;
- uint8x8x4_t _dst1;
- uint8x8x4_t _dst2;
- uint8x8x4_t _dst3;
- uint8x8x4_t _dst4;
- uint8x8x4_t _dst5;
- uint8x8x4_t _dst6;
- uint8x8x4_t _dst7;
-
- _dst0.val[0] = vreinterpret_u8_u32(_src04ttt_r.val[1]);
- _dst1.val[0] = vreinterpret_u8_u32(_src15ttt_r.val[1]);
- _dst2.val[0] = vreinterpret_u8_u32(_src26ttt_r.val[1]);
- _dst3.val[0] = vreinterpret_u8_u32(_src37ttt_r.val[1]);
- _dst4.val[0] = vreinterpret_u8_u32(_src04ttt_r.val[0]);
- _dst5.val[0] = vreinterpret_u8_u32(_src15ttt_r.val[0]);
- _dst6.val[0] = vreinterpret_u8_u32(_src26ttt_r.val[0]);
- _dst7.val[0] = vreinterpret_u8_u32(_src37ttt_r.val[0]);
-
- _dst0.val[1] = vreinterpret_u8_u32(_src04ttt_g.val[1]);
- _dst1.val[1] = vreinterpret_u8_u32(_src15ttt_g.val[1]);
- _dst2.val[1] = vreinterpret_u8_u32(_src26ttt_g.val[1]);
- _dst3.val[1] = vreinterpret_u8_u32(_src37ttt_g.val[1]);
- _dst4.val[1] = vreinterpret_u8_u32(_src04ttt_g.val[0]);
- _dst5.val[1] = vreinterpret_u8_u32(_src15ttt_g.val[0]);
- _dst6.val[1] = vreinterpret_u8_u32(_src26ttt_g.val[0]);
- _dst7.val[1] = vreinterpret_u8_u32(_src37ttt_g.val[0]);
-
- _dst0.val[2] = vreinterpret_u8_u32(_src04ttt_b.val[1]);
- _dst1.val[2] = vreinterpret_u8_u32(_src15ttt_b.val[1]);
- _dst2.val[2] = vreinterpret_u8_u32(_src26ttt_b.val[1]);
- _dst3.val[2] = vreinterpret_u8_u32(_src37ttt_b.val[1]);
- _dst4.val[2] = vreinterpret_u8_u32(_src04ttt_b.val[0]);
- _dst5.val[2] = vreinterpret_u8_u32(_src15ttt_b.val[0]);
- _dst6.val[2] = vreinterpret_u8_u32(_src26ttt_b.val[0]);
- _dst7.val[2] = vreinterpret_u8_u32(_src37ttt_b.val[0]);
-
- _dst0.val[3] = vreinterpret_u8_u32(_src04ttt_a.val[1]);
- _dst1.val[3] = vreinterpret_u8_u32(_src15ttt_a.val[1]);
- _dst2.val[3] = vreinterpret_u8_u32(_src26ttt_a.val[1]);
- _dst3.val[3] = vreinterpret_u8_u32(_src37ttt_a.val[1]);
- _dst4.val[3] = vreinterpret_u8_u32(_src04ttt_a.val[0]);
- _dst5.val[3] = vreinterpret_u8_u32(_src15ttt_a.val[0]);
- _dst6.val[3] = vreinterpret_u8_u32(_src26ttt_a.val[0]);
- _dst7.val[3] = vreinterpret_u8_u32(_src37ttt_a.val[0]);
-
- vst4_u8(dst0, _dst7);
- vst4_u8(dst1, _dst6);
- vst4_u8(dst0 + dst_step, _dst5);
- vst4_u8(dst1 + dst_step, _dst4);
- vst4_u8(dst0 + 2 * dst_step, _dst3);
- vst4_u8(dst1 + 2 * dst_step, _dst2);
- vst4_u8(dst0 + 3 * dst_step, _dst1);
- vst4_u8(dst1 + 3 * dst_step, _dst0);
-
- src0 += 4 * 8;
- src1 += 4 * 8;
-
- dst0 += 4 * dst_step;
- dst1 += 4 * dst_step;
- }
- #else
- if (nn > 0)
- {
- asm volatile(
- "0: \n"
- "pld [%1, #256] \n"
- "vld4.u8 {d0-d3}, [%1], %10 \n"
-
- "pld [%2, #256] \n"
- "vld4.u8 {d4-d7}, [%2], %10 \n"
-
- "pld [%1, #256] \n"
- "vld4.u8 {d8-d11}, [%1], %10 \n"
-
- "vtrn.u8 q2, q0 \n" // _src01t_r
- "vtrn.u8 q3, q1 \n"
-
- "pld [%2, #256] \n"
- "vld4.u8 {d12-d15}, [%2], %10\n"
-
- "pld [%1, #256] \n"
- "vld4.u8 {d16-d19}, [%1], %10\n"
-
- "vtrn.u8 q6, q4 \n" // _src23t_r
- "vtrn.u8 q7, q5 \n"
-
- "pld [%2, #256] \n"
- "vld4.u8 {d20-d23}, [%2], %10\n"
-
- "pld [%1, #256] \n"
- "vld4.u8 {d24-d27}, [%1], %10\n"
-
- "vtrn.u8 q10, q8 \n" // _src45t_r
- "vtrn.u8 q11, q9 \n"
-
- "pld [%2, #256] \n"
- "vld4.u8 {d28-d31}, [%2], %10\n"
-
- "vtrn.u8 q14, q12 \n" // _src67t_r
- "vtrn.u8 q15, q13 \n"
-
- "sub %1, %1, %10, lsl #2 \n" // restore src0
-
- "vtrn.u16 q4, q0 \n" // _src02tt_r
- "vtrn.u16 q5, q1 \n"
-
- "sub %2, %2, %10, lsl #2 \n" // restore src1
-
- "vtrn.u16 q6, q2 \n" // _src13tt_r
- "vtrn.u16 q7, q3 \n"
-
- "add %1, #32 \n" // src0 += 32
-
- "vtrn.u16 q12, q8 \n" // _src46tt_r
- "vtrn.u16 q13, q9 \n"
-
- "add %2, #32 \n" // src1 += 32
-
- "vtrn.u16 q14, q10 \n" // _src57tt_r
- "vtrn.u16 q15, q11 \n"
-
- "vtrn.u32 q12, q4 \n" // _src26ttt_r
- "vtrn.u32 q13, q5 \n"
-
- "vtrn.u32 q14, q6 \n" // _src37ttt_r
- "vst4.u8 {d24-d27}, [%4], %11\n"
- "vtrn.u32 q15, q7 \n"
-
- "vtrn.u32 q8, q0 \n" // _src04ttt_r
- "vst4.u8 {d28-d31}, [%3], %11\n"
- "vtrn.u32 q9, q1 \n"
-
- "vtrn.u32 q10, q2 \n" // _src15ttt_r
- "vst4.u8 {d16-d19}, [%4], %11\n"
- "vtrn.u32 q11, q3 \n"
-
- "subs %0, #1 \n"
-
- "vst4.u8 {d8-d11}, [%4], %11 \n"
- "vst4.u8 {d20-d23}, [%3], %11\n"
- "vst4.u8 {d12-d15}, [%3], %11\n"
- "vst4.u8 {d0-d3}, [%4], %11 \n"
- "vst4.u8 {d4-d7}, [%3], %11 \n"
-
- "bne 0b \n"
- : "=r"(nn), // %0
- "=r"(src0), // %1
- "=r"(src1), // %2
- "=r"(dst0), // %3
- "=r"(dst1) // %4
- : "0"(nn),
- "1"(src0),
- "2"(src1),
- "3"(dst0),
- "4"(dst1),
- "r"(src_step), // %10
- "r"(dst_step) // %11
- : "cc", "memory", "q0", "q1", "q2", "q3", "q4", "q5", "q6", "q7", "q8", "q9", "q10", "q11", "q12", "q13", "q14", "q15");
- }
- #endif // __aarch64__
- for (; remain > 0; remain--)
- {
- dst0[0] = src1[0 + 3 * src_step];
- dst0[1] = src1[1 + 3 * src_step];
- dst0[2] = src1[2 + 3 * src_step];
- dst0[3] = src1[3 + 3 * src_step];
- dst0[4] = src0[0 + 3 * src_step];
- dst0[5] = src0[1 + 3 * src_step];
- dst0[6] = src0[2 + 3 * src_step];
- dst0[7] = src0[3 + 3 * src_step];
- dst0[8] = src1[0 + 2 * src_step];
- dst0[9] = src1[1 + 2 * src_step];
- dst0[10] = src1[2 + 2 * src_step];
- dst0[11] = src1[3 + 2 * src_step];
- dst0[12] = src0[0 + 2 * src_step];
- dst0[13] = src0[1 + 2 * src_step];
- dst0[14] = src0[2 + 2 * src_step];
- dst0[15] = src0[3 + 2 * src_step];
- dst0[16] = src1[0 + src_step];
- dst0[17] = src1[1 + src_step];
- dst0[18] = src1[2 + src_step];
- dst0[19] = src1[3 + src_step];
- dst0[20] = src0[0 + src_step];
- dst0[21] = src0[1 + src_step];
- dst0[22] = src0[2 + src_step];
- dst0[23] = src0[3 + src_step];
- dst0[24] = src1[0];
- dst0[25] = src1[1];
- dst0[26] = src1[2];
- dst0[27] = src1[3];
- dst0[28] = src0[0];
- dst0[29] = src0[1];
- dst0[30] = src0[2];
- dst0[31] = src0[3];
-
- src0 += 4;
- src1 += 4;
-
- dst0 += stride;
- }
-
- src0 += srcwgap + 7 * srcstride;
- }
- #endif // __ARM_NEON
- for (; y < srch; y++)
- {
- unsigned char* dst0 = dstend - y * 4 - 4;
-
- int x = 0;
- for (; x < srcw; x++)
- {
- dst0[0] = src0[0];
- dst0[1] = src0[1];
- dst0[2] = src0[2];
- dst0[3] = src0[3];
-
- src0 += 4;
- dst0 += stride;
- }
-
- src0 += srcwgap;
- }
- }
-
- static void kanna_rotate_7_c1(const unsigned char* src, int srcw, int srch, int srcstride, unsigned char* dst, int w, int h, int stride)
- {
- const int srcwgap = srcstride - srcw;
-
- // point to the last dst pixel
- unsigned char* dstend = dst + stride * (h - 1) + w;
-
- const unsigned char* src0 = src;
-
- int y = 0;
- #if __ARM_NEON
- for (; y + 7 < srch; y += 8)
- {
- const unsigned char* src1 = src0 + srcstride;
-
- unsigned char* dst6 = dstend - y - 8 - stride;
- unsigned char* dst7 = dstend - y - 8;
-
- int src_step = 2 * srcstride;
- int dst_step = -2 * stride;
-
- int nn = srcw >> 3;
- int remain = srcw - (nn << 3);
-
- #if !NCNN_GNU_INLINE_ASM || __aarch64__
- for (; nn > 0; nn--)
- {
- uint8x8_t _src0 = vld1_u8(src0);
- uint8x8_t _src1 = vld1_u8(src1);
-
- uint8x8_t _src2 = vld1_u8(src0 + src_step);
- uint8x8_t _src3 = vld1_u8(src1 + src_step);
-
- uint8x8_t _src4 = vld1_u8(src0 + 2 * src_step);
- uint8x8_t _src5 = vld1_u8(src1 + 2 * src_step);
-
- uint8x8_t _src6 = vld1_u8(src0 + 3 * src_step);
- uint8x8_t _src7 = vld1_u8(src1 + 3 * src_step);
-
- uint8x8x2_t _src01t_r = vtrn_u8(_src1, _src0);
- uint8x8x2_t _src23t_r = vtrn_u8(_src3, _src2);
- uint8x8x2_t _src45t_r = vtrn_u8(_src5, _src4);
- uint8x8x2_t _src67t_r = vtrn_u8(_src7, _src6);
-
- uint16x4x2_t _src02tt_r = vtrn_u16(vreinterpret_u16_u8(_src23t_r.val[1]), vreinterpret_u16_u8(_src01t_r.val[1]));
- uint16x4x2_t _src13tt_r = vtrn_u16(vreinterpret_u16_u8(_src23t_r.val[0]), vreinterpret_u16_u8(_src01t_r.val[0]));
- uint16x4x2_t _src46tt_r = vtrn_u16(vreinterpret_u16_u8(_src67t_r.val[1]), vreinterpret_u16_u8(_src45t_r.val[1]));
- uint16x4x2_t _src57tt_r = vtrn_u16(vreinterpret_u16_u8(_src67t_r.val[0]), vreinterpret_u16_u8(_src45t_r.val[0]));
-
- uint32x2x2_t _src04ttt_r = vtrn_u32(vreinterpret_u32_u16(_src46tt_r.val[1]), vreinterpret_u32_u16(_src02tt_r.val[1]));
- uint32x2x2_t _src15ttt_r = vtrn_u32(vreinterpret_u32_u16(_src57tt_r.val[1]), vreinterpret_u32_u16(_src13tt_r.val[1]));
- uint32x2x2_t _src26ttt_r = vtrn_u32(vreinterpret_u32_u16(_src46tt_r.val[0]), vreinterpret_u32_u16(_src02tt_r.val[0]));
- uint32x2x2_t _src37ttt_r = vtrn_u32(vreinterpret_u32_u16(_src57tt_r.val[0]), vreinterpret_u32_u16(_src13tt_r.val[0]));
-
- uint8x8_t _dst0 = vreinterpret_u8_u32(_src04ttt_r.val[1]);
- uint8x8_t _dst1 = vreinterpret_u8_u32(_src15ttt_r.val[1]);
- uint8x8_t _dst2 = vreinterpret_u8_u32(_src26ttt_r.val[1]);
- uint8x8_t _dst3 = vreinterpret_u8_u32(_src37ttt_r.val[1]);
- uint8x8_t _dst4 = vreinterpret_u8_u32(_src04ttt_r.val[0]);
- uint8x8_t _dst5 = vreinterpret_u8_u32(_src15ttt_r.val[0]);
- uint8x8_t _dst6 = vreinterpret_u8_u32(_src26ttt_r.val[0]);
- uint8x8_t _dst7 = vreinterpret_u8_u32(_src37ttt_r.val[0]);
-
- vst1_u8(dst7, _dst7);
- vst1_u8(dst6, _dst6);
- vst1_u8(dst7 + dst_step, _dst5);
- vst1_u8(dst6 + dst_step, _dst4);
- vst1_u8(dst7 + 2 * dst_step, _dst3);
- vst1_u8(dst6 + 2 * dst_step, _dst2);
- vst1_u8(dst7 + 3 * dst_step, _dst1);
- vst1_u8(dst6 + 3 * dst_step, _dst0);
-
- src0 += 8;
- src1 += 8;
-
- dst7 += 4 * dst_step;
- dst6 += 4 * dst_step;
- }
- #else
- if (nn > 0)
- {
- asm volatile(
- "0: \n"
- "pld [%1, #64] \n"
- "vld1.u8 {d0}, [%1], %10 \n"
-
- "pld [%2, #64] \n"
- "vld1.u8 {d1}, [%2], %10 \n"
-
- "pld [%1, #64] \n"
- "vld1.u8 {d2}, [%1], %10 \n"
-
- "vtrn.u8 d1, d0 \n" // _src01t_r
-
- "pld [%2, #64] \n"
- "vld1.u8 {d3}, [%2], %10 \n"
-
- "pld [%1, #64] \n"
- "vld1.u8 {d4}, [%1], %10 \n"
-
- "vtrn.u8 d3, d2 \n" // _src23t_r
-
- "pld [%2, #64] \n"
- "vld1.u8 {d5}, [%2], %10 \n"
-
- "pld [%1, #64] \n"
- "vld1.u8 {d6}, [%1], %10 \n"
-
- "vtrn.u8 d5, d4 \n" // _src45t_r
-
- "pld [%2, #64] \n"
- "vld1.u8 {d7}, [%2], %10 \n"
-
- "vtrn.u8 d7, d6 \n" // _src67t_r
-
- "sub %1, %1, %10, lsl #2 \n" // restore src0
-
- "vtrn.u16 q1, q0 \n" // _src02tt_r _src13tt_r
-
- "sub %2, %2, %10, lsl #2 \n" // restore src1
-
- "vtrn.u16 q3, q2 \n" // _src46tt_r _src57tt_r
-
- "add %1, #8 \n" // src0 += 8
-
- "vtrn.u32 q3, q1 \n" // _src26ttt_r _src37ttt_r
-
- "add %2, #8 \n" // src1 += 8
-
- "vtrn.u32 q2, q0 \n" // _src04ttt_r _src15ttt_r
- "vst1.u8 {d6}, [%4], %11 \n"
- "vst1.u8 {d7}, [%3], %11 \n"
-
- "subs %0, #1 \n"
-
- "vst1.u8 {d4}, [%4], %11 \n"
- "vst1.u8 {d5}, [%3], %11 \n"
- "vst1.u8 {d2}, [%4], %11 \n"
- "vst1.u8 {d3}, [%3], %11 \n"
- "vst1.u8 {d0}, [%4], %11 \n"
- "vst1.u8 {d1}, [%3], %11 \n"
-
- "bne 0b \n"
- : "=r"(nn), // %0
- "=r"(src0), // %1
- "=r"(src1), // %2
- "=r"(dst7), // %3
- "=r"(dst6) // %4
- : "0"(nn),
- "1"(src0),
- "2"(src1),
- "3"(dst7),
- "4"(dst6),
- "r"(src_step), // %10
- "r"(dst_step) // %11
- : "cc", "memory", "q0", "q1", "q2", "q3");
- }
- #endif // __aarch64__
- for (; remain > 0; remain--)
- {
- dst7[0] = src1[0 + 3 * src_step];
- dst7[1] = src0[0 + 3 * src_step];
- dst7[2] = src1[0 + 2 * src_step];
- dst7[3] = src0[0 + 2 * src_step];
- dst7[4] = src1[0 + src_step];
- dst7[5] = src0[0 + src_step];
- dst7[6] = src1[0];
- dst7[7] = src0[0];
-
- src0 += 1;
- src1 += 1;
-
- dst7 -= stride;
- }
-
- src0 += srcwgap + 7 * srcstride;
- }
- #endif // __ARM_NEON
- for (; y < srch; y++)
- {
- unsigned char* dst0 = dstend - y - 1;
-
- int x = 0;
- for (; x < srcw; x++)
- {
- *dst0 = *src0;
-
- src0 += 1;
- dst0 -= stride;
- }
-
- src0 += srcwgap;
- }
- }
-
- static void kanna_rotate_7_c2(const unsigned char* src, int srcw, int srch, int srcstride, unsigned char* dst, int w, int h, int stride)
- {
- const int srcwgap = srcstride - srcw * 2;
-
- // point to the last dst pixel
- unsigned char* dstend = dst + stride * (h - 1) + w * 2;
-
- const unsigned char* src0 = src;
-
- int y = 0;
- #if __ARM_NEON
- for (; y + 7 < srch; y += 8)
- {
- const unsigned char* src1 = src0 + srcstride;
-
- unsigned char* dst6 = dstend - y * 2 - 8 * 2 - stride;
- unsigned char* dst7 = dstend - y * 2 - 8 * 2;
-
- int src_step = 2 * srcstride;
- int dst_step = -2 * stride;
-
- int nn = srcw >> 3;
- int remain = srcw - (nn << 3);
-
- #if !NCNN_GNU_INLINE_ASM || __aarch64__
- for (; nn > 0; nn--)
- {
- uint8x8x2_t _src0 = vld2_u8(src0);
- uint8x8x2_t _src1 = vld2_u8(src1);
-
- uint8x8x2_t _src2 = vld2_u8(src0 + src_step);
- uint8x8x2_t _src3 = vld2_u8(src1 + src_step);
-
- uint8x8x2_t _src4 = vld2_u8(src0 + 2 * src_step);
- uint8x8x2_t _src5 = vld2_u8(src1 + 2 * src_step);
-
- uint8x8x2_t _src6 = vld2_u8(src0 + 3 * src_step);
- uint8x8x2_t _src7 = vld2_u8(src1 + 3 * src_step);
-
- uint8x8x2_t _src01t_r = vtrn_u8(_src1.val[0], _src0.val[0]);
- uint8x8x2_t _src23t_r = vtrn_u8(_src3.val[0], _src2.val[0]);
- uint8x8x2_t _src45t_r = vtrn_u8(_src5.val[0], _src4.val[0]);
- uint8x8x2_t _src67t_r = vtrn_u8(_src7.val[0], _src6.val[0]);
-
- uint8x8x2_t _src01t_g = vtrn_u8(_src1.val[1], _src0.val[1]);
- uint8x8x2_t _src23t_g = vtrn_u8(_src3.val[1], _src2.val[1]);
- uint8x8x2_t _src45t_g = vtrn_u8(_src5.val[1], _src4.val[1]);
- uint8x8x2_t _src67t_g = vtrn_u8(_src7.val[1], _src6.val[1]);
-
- uint16x4x2_t _src02tt_r = vtrn_u16(vreinterpret_u16_u8(_src23t_r.val[1]), vreinterpret_u16_u8(_src01t_r.val[1]));
- uint16x4x2_t _src13tt_r = vtrn_u16(vreinterpret_u16_u8(_src23t_r.val[0]), vreinterpret_u16_u8(_src01t_r.val[0]));
- uint16x4x2_t _src46tt_r = vtrn_u16(vreinterpret_u16_u8(_src67t_r.val[1]), vreinterpret_u16_u8(_src45t_r.val[1]));
- uint16x4x2_t _src57tt_r = vtrn_u16(vreinterpret_u16_u8(_src67t_r.val[0]), vreinterpret_u16_u8(_src45t_r.val[0]));
-
- uint16x4x2_t _src02tt_g = vtrn_u16(vreinterpret_u16_u8(_src23t_g.val[1]), vreinterpret_u16_u8(_src01t_g.val[1]));
- uint16x4x2_t _src13tt_g = vtrn_u16(vreinterpret_u16_u8(_src23t_g.val[0]), vreinterpret_u16_u8(_src01t_g.val[0]));
- uint16x4x2_t _src46tt_g = vtrn_u16(vreinterpret_u16_u8(_src67t_g.val[1]), vreinterpret_u16_u8(_src45t_g.val[1]));
- uint16x4x2_t _src57tt_g = vtrn_u16(vreinterpret_u16_u8(_src67t_g.val[0]), vreinterpret_u16_u8(_src45t_g.val[0]));
-
- uint32x2x2_t _src04ttt_r = vtrn_u32(vreinterpret_u32_u16(_src46tt_r.val[1]), vreinterpret_u32_u16(_src02tt_r.val[1]));
- uint32x2x2_t _src15ttt_r = vtrn_u32(vreinterpret_u32_u16(_src57tt_r.val[1]), vreinterpret_u32_u16(_src13tt_r.val[1]));
- uint32x2x2_t _src26ttt_r = vtrn_u32(vreinterpret_u32_u16(_src46tt_r.val[0]), vreinterpret_u32_u16(_src02tt_r.val[0]));
- uint32x2x2_t _src37ttt_r = vtrn_u32(vreinterpret_u32_u16(_src57tt_r.val[0]), vreinterpret_u32_u16(_src13tt_r.val[0]));
-
- uint32x2x2_t _src04ttt_g = vtrn_u32(vreinterpret_u32_u16(_src46tt_g.val[1]), vreinterpret_u32_u16(_src02tt_g.val[1]));
- uint32x2x2_t _src15ttt_g = vtrn_u32(vreinterpret_u32_u16(_src57tt_g.val[1]), vreinterpret_u32_u16(_src13tt_g.val[1]));
- uint32x2x2_t _src26ttt_g = vtrn_u32(vreinterpret_u32_u16(_src46tt_g.val[0]), vreinterpret_u32_u16(_src02tt_g.val[0]));
- uint32x2x2_t _src37ttt_g = vtrn_u32(vreinterpret_u32_u16(_src57tt_g.val[0]), vreinterpret_u32_u16(_src13tt_g.val[0]));
-
- uint8x8x2_t _dst0;
- uint8x8x2_t _dst1;
- uint8x8x2_t _dst2;
- uint8x8x2_t _dst3;
- uint8x8x2_t _dst4;
- uint8x8x2_t _dst5;
- uint8x8x2_t _dst6;
- uint8x8x2_t _dst7;
-
- _dst0.val[0] = vreinterpret_u8_u32(_src04ttt_r.val[1]);
- _dst1.val[0] = vreinterpret_u8_u32(_src15ttt_r.val[1]);
- _dst2.val[0] = vreinterpret_u8_u32(_src26ttt_r.val[1]);
- _dst3.val[0] = vreinterpret_u8_u32(_src37ttt_r.val[1]);
- _dst4.val[0] = vreinterpret_u8_u32(_src04ttt_r.val[0]);
- _dst5.val[0] = vreinterpret_u8_u32(_src15ttt_r.val[0]);
- _dst6.val[0] = vreinterpret_u8_u32(_src26ttt_r.val[0]);
- _dst7.val[0] = vreinterpret_u8_u32(_src37ttt_r.val[0]);
-
- _dst0.val[1] = vreinterpret_u8_u32(_src04ttt_g.val[1]);
- _dst1.val[1] = vreinterpret_u8_u32(_src15ttt_g.val[1]);
- _dst2.val[1] = vreinterpret_u8_u32(_src26ttt_g.val[1]);
- _dst3.val[1] = vreinterpret_u8_u32(_src37ttt_g.val[1]);
- _dst4.val[1] = vreinterpret_u8_u32(_src04ttt_g.val[0]);
- _dst5.val[1] = vreinterpret_u8_u32(_src15ttt_g.val[0]);
- _dst6.val[1] = vreinterpret_u8_u32(_src26ttt_g.val[0]);
- _dst7.val[1] = vreinterpret_u8_u32(_src37ttt_g.val[0]);
-
- vst2_u8(dst7, _dst7);
- vst2_u8(dst6, _dst6);
- vst2_u8(dst7 + dst_step, _dst5);
- vst2_u8(dst6 + dst_step, _dst4);
- vst2_u8(dst7 + 2 * dst_step, _dst3);
- vst2_u8(dst6 + 2 * dst_step, _dst2);
- vst2_u8(dst7 + 3 * dst_step, _dst1);
- vst2_u8(dst6 + 3 * dst_step, _dst0);
-
- src0 += 2 * 8;
- src1 += 2 * 8;
-
- dst7 += 4 * dst_step;
- dst6 += 4 * dst_step;
- }
- #else
- if (nn > 0)
- {
- asm volatile(
- "0: \n"
- "pld [%1, #128] \n"
- "vld2.u8 {d0-d1}, [%1], %10 \n"
-
- "pld [%2, #128] \n"
- "vld2.u8 {d2-d3}, [%2], %10 \n"
-
- "pld [%1, #128] \n"
- "vld2.u8 {d4-d5}, [%1], %10 \n"
-
- "vtrn.u8 q1, q0 \n" // _src01t_r
-
- "pld [%2, #128] \n"
- "vld2.u8 {d6-d7}, [%2], %10 \n"
-
- "pld [%1, #128] \n"
- "vld2.u8 {d16-d17}, [%1], %10\n"
-
- "vtrn.u8 q3, q2 \n" // _src23t_r
-
- "pld [%2, #128] \n"
- "vld2.u8 {d18-d19}, [%2], %10\n"
-
- "pld [%1, #128] \n"
- "vld2.u8 {d20-d21}, [%1], %10\n"
-
- "vtrn.u8 q9, q8 \n" // _src45t_r
-
- "pld [%2, #128] \n"
- "vld2.u8 {d22-d23}, [%2], %10\n"
-
- "vtrn.u8 q11, q10 \n" // _src67t_r
-
- "sub %1, %1, %10, lsl #2 \n" // restore src0
-
- "vtrn.u16 q2, q0 \n" // _src02tt_r
-
- "sub %2, %2, %10, lsl #2 \n" // restore src1
-
- "vtrn.u16 q3, q1 \n" // _src13tt_r
-
- "add %1, #16 \n" // src0 += 16
-
- "vtrn.u16 q10, q8 \n" // _src46tt_r
-
- "add %2, #16 \n" // src1 += 16
-
- "vtrn.u16 q11, q9 \n" // _src57tt_r
-
- "vtrn.u32 q10, q2 \n" // _src26ttt_r
-
- "vtrn.u32 q11, q3 \n" // _src37ttt_r
- "vst2.u8 {d20-d21}, [%4], %11\n"
-
- "vtrn.u32 q8, q0 \n" // _src04ttt_r
- "vst2.u8 {d22-d23}, [%3], %11\n"
-
- "vtrn.u32 q9, q1 \n" // _src15ttt_r
- "vst2.u8 {d16-d17}, [%4], %11\n"
-
- "subs %0, #1 \n"
-
- "vst2.u8 {d4-d5}, [%4], %11 \n"
- "vst2.u8 {d18-d19}, [%3], %11\n"
- "vst2.u8 {d6-d7}, [%3], %11 \n"
- "vst2.u8 {d0-d1}, [%4], %11 \n"
- "vst2.u8 {d2-d3}, [%3], %11 \n"
-
- "bne 0b \n"
- : "=r"(nn), // %0
- "=r"(src0), // %1
- "=r"(src1), // %2
- "=r"(dst7), // %3
- "=r"(dst6) // %4
- : "0"(nn),
- "1"(src0),
- "2"(src1),
- "3"(dst7),
- "4"(dst6),
- "r"(src_step), // %10
- "r"(dst_step) // %11
- : "cc", "memory", "q0", "q1", "q2", "q3", "q8", "q9", "q10", "q11");
- }
- #endif // __aarch64__
- for (; remain > 0; remain--)
- {
- dst7[0] = src1[0 + 3 * src_step];
- dst7[1] = src1[1 + 3 * src_step];
- dst7[2] = src0[0 + 3 * src_step];
- dst7[3] = src0[1 + 3 * src_step];
- dst7[4] = src1[0 + 2 * src_step];
- dst7[5] = src1[1 + 2 * src_step];
- dst7[6] = src0[0 + 2 * src_step];
- dst7[7] = src0[1 + 2 * src_step];
- dst7[8] = src1[0 + src_step];
- dst7[9] = src1[1 + src_step];
- dst7[10] = src0[0 + src_step];
- dst7[11] = src0[1 + src_step];
- dst7[12] = src1[0];
- dst7[13] = src1[1];
- dst7[14] = src0[0];
- dst7[15] = src0[1];
-
- src0 += 2;
- src1 += 2;
-
- dst7 -= stride;
- }
-
- src0 += srcwgap + 7 * srcstride;
- }
- #endif // __ARM_NEON
- for (; y < srch; y++)
- {
- unsigned char* dst0 = dstend - y * 2 - 2;
-
- int x = 0;
- for (; x < srcw; x++)
- {
- dst0[0] = src0[0];
- dst0[1] = src0[1];
-
- src0 += 2;
- dst0 -= stride;
- }
-
- src0 += srcwgap;
- }
- }
-
- static void kanna_rotate_7_c3(const unsigned char* src, int srcw, int srch, int srcstride, unsigned char* dst, int w, int h, int stride)
- {
- const int srcwgap = srcstride - srcw * 3;
-
- // point to the last dst pixel
- unsigned char* dstend = dst + stride * (h - 1) + w * 3;
-
- const unsigned char* src0 = src;
-
- int y = 0;
- #if __ARM_NEON
- for (; y + 7 < srch; y += 8)
- {
- const unsigned char* src1 = src0 + srcstride;
-
- unsigned char* dst6 = dstend - y * 3 - 8 * 3 - stride;
- unsigned char* dst7 = dstend - y * 3 - 8 * 3;
-
- int src_step = 2 * srcstride;
- int dst_step = -2 * stride;
-
- int nn = srcw >> 3;
- int remain = srcw - (nn << 3);
-
- #if !NCNN_GNU_INLINE_ASM || __aarch64__
- for (; nn > 0; nn--)
- {
- uint8x8x3_t _src0 = vld3_u8(src0);
- uint8x8x3_t _src1 = vld3_u8(src1);
-
- uint8x8x3_t _src2 = vld3_u8(src0 + src_step);
- uint8x8x3_t _src3 = vld3_u8(src1 + src_step);
-
- uint8x8x3_t _src4 = vld3_u8(src0 + 2 * src_step);
- uint8x8x3_t _src5 = vld3_u8(src1 + 2 * src_step);
-
- uint8x8x3_t _src6 = vld3_u8(src0 + 3 * src_step);
- uint8x8x3_t _src7 = vld3_u8(src1 + 3 * src_step);
-
- uint8x8x2_t _src01t_r = vtrn_u8(_src1.val[0], _src0.val[0]);
- uint8x8x2_t _src23t_r = vtrn_u8(_src3.val[0], _src2.val[0]);
- uint8x8x2_t _src45t_r = vtrn_u8(_src5.val[0], _src4.val[0]);
- uint8x8x2_t _src67t_r = vtrn_u8(_src7.val[0], _src6.val[0]);
-
- uint8x8x2_t _src01t_g = vtrn_u8(_src1.val[1], _src0.val[1]);
- uint8x8x2_t _src23t_g = vtrn_u8(_src3.val[1], _src2.val[1]);
- uint8x8x2_t _src45t_g = vtrn_u8(_src5.val[1], _src4.val[1]);
- uint8x8x2_t _src67t_g = vtrn_u8(_src7.val[1], _src6.val[1]);
-
- uint8x8x2_t _src01t_b = vtrn_u8(_src1.val[2], _src0.val[2]);
- uint8x8x2_t _src23t_b = vtrn_u8(_src3.val[2], _src2.val[2]);
- uint8x8x2_t _src45t_b = vtrn_u8(_src5.val[2], _src4.val[2]);
- uint8x8x2_t _src67t_b = vtrn_u8(_src7.val[2], _src6.val[2]);
-
- uint16x4x2_t _src02tt_r = vtrn_u16(vreinterpret_u16_u8(_src23t_r.val[1]), vreinterpret_u16_u8(_src01t_r.val[1]));
- uint16x4x2_t _src13tt_r = vtrn_u16(vreinterpret_u16_u8(_src23t_r.val[0]), vreinterpret_u16_u8(_src01t_r.val[0]));
- uint16x4x2_t _src46tt_r = vtrn_u16(vreinterpret_u16_u8(_src67t_r.val[1]), vreinterpret_u16_u8(_src45t_r.val[1]));
- uint16x4x2_t _src57tt_r = vtrn_u16(vreinterpret_u16_u8(_src67t_r.val[0]), vreinterpret_u16_u8(_src45t_r.val[0]));
-
- uint16x4x2_t _src02tt_g = vtrn_u16(vreinterpret_u16_u8(_src23t_g.val[1]), vreinterpret_u16_u8(_src01t_g.val[1]));
- uint16x4x2_t _src13tt_g = vtrn_u16(vreinterpret_u16_u8(_src23t_g.val[0]), vreinterpret_u16_u8(_src01t_g.val[0]));
- uint16x4x2_t _src46tt_g = vtrn_u16(vreinterpret_u16_u8(_src67t_g.val[1]), vreinterpret_u16_u8(_src45t_g.val[1]));
- uint16x4x2_t _src57tt_g = vtrn_u16(vreinterpret_u16_u8(_src67t_g.val[0]), vreinterpret_u16_u8(_src45t_g.val[0]));
-
- uint16x4x2_t _src02tt_b = vtrn_u16(vreinterpret_u16_u8(_src23t_b.val[1]), vreinterpret_u16_u8(_src01t_b.val[1]));
- uint16x4x2_t _src13tt_b = vtrn_u16(vreinterpret_u16_u8(_src23t_b.val[0]), vreinterpret_u16_u8(_src01t_b.val[0]));
- uint16x4x2_t _src46tt_b = vtrn_u16(vreinterpret_u16_u8(_src67t_b.val[1]), vreinterpret_u16_u8(_src45t_b.val[1]));
- uint16x4x2_t _src57tt_b = vtrn_u16(vreinterpret_u16_u8(_src67t_b.val[0]), vreinterpret_u16_u8(_src45t_b.val[0]));
-
- uint32x2x2_t _src04ttt_r = vtrn_u32(vreinterpret_u32_u16(_src46tt_r.val[1]), vreinterpret_u32_u16(_src02tt_r.val[1]));
- uint32x2x2_t _src15ttt_r = vtrn_u32(vreinterpret_u32_u16(_src57tt_r.val[1]), vreinterpret_u32_u16(_src13tt_r.val[1]));
- uint32x2x2_t _src26ttt_r = vtrn_u32(vreinterpret_u32_u16(_src46tt_r.val[0]), vreinterpret_u32_u16(_src02tt_r.val[0]));
- uint32x2x2_t _src37ttt_r = vtrn_u32(vreinterpret_u32_u16(_src57tt_r.val[0]), vreinterpret_u32_u16(_src13tt_r.val[0]));
-
- uint32x2x2_t _src04ttt_g = vtrn_u32(vreinterpret_u32_u16(_src46tt_g.val[1]), vreinterpret_u32_u16(_src02tt_g.val[1]));
- uint32x2x2_t _src15ttt_g = vtrn_u32(vreinterpret_u32_u16(_src57tt_g.val[1]), vreinterpret_u32_u16(_src13tt_g.val[1]));
- uint32x2x2_t _src26ttt_g = vtrn_u32(vreinterpret_u32_u16(_src46tt_g.val[0]), vreinterpret_u32_u16(_src02tt_g.val[0]));
- uint32x2x2_t _src37ttt_g = vtrn_u32(vreinterpret_u32_u16(_src57tt_g.val[0]), vreinterpret_u32_u16(_src13tt_g.val[0]));
-
- uint32x2x2_t _src04ttt_b = vtrn_u32(vreinterpret_u32_u16(_src46tt_b.val[1]), vreinterpret_u32_u16(_src02tt_b.val[1]));
- uint32x2x2_t _src15ttt_b = vtrn_u32(vreinterpret_u32_u16(_src57tt_b.val[1]), vreinterpret_u32_u16(_src13tt_b.val[1]));
- uint32x2x2_t _src26ttt_b = vtrn_u32(vreinterpret_u32_u16(_src46tt_b.val[0]), vreinterpret_u32_u16(_src02tt_b.val[0]));
- uint32x2x2_t _src37ttt_b = vtrn_u32(vreinterpret_u32_u16(_src57tt_b.val[0]), vreinterpret_u32_u16(_src13tt_b.val[0]));
-
- uint8x8x3_t _dst0;
- uint8x8x3_t _dst1;
- uint8x8x3_t _dst2;
- uint8x8x3_t _dst3;
- uint8x8x3_t _dst4;
- uint8x8x3_t _dst5;
- uint8x8x3_t _dst6;
- uint8x8x3_t _dst7;
-
- _dst0.val[0] = vreinterpret_u8_u32(_src04ttt_r.val[1]);
- _dst1.val[0] = vreinterpret_u8_u32(_src15ttt_r.val[1]);
- _dst2.val[0] = vreinterpret_u8_u32(_src26ttt_r.val[1]);
- _dst3.val[0] = vreinterpret_u8_u32(_src37ttt_r.val[1]);
- _dst4.val[0] = vreinterpret_u8_u32(_src04ttt_r.val[0]);
- _dst5.val[0] = vreinterpret_u8_u32(_src15ttt_r.val[0]);
- _dst6.val[0] = vreinterpret_u8_u32(_src26ttt_r.val[0]);
- _dst7.val[0] = vreinterpret_u8_u32(_src37ttt_r.val[0]);
-
- _dst0.val[1] = vreinterpret_u8_u32(_src04ttt_g.val[1]);
- _dst1.val[1] = vreinterpret_u8_u32(_src15ttt_g.val[1]);
- _dst2.val[1] = vreinterpret_u8_u32(_src26ttt_g.val[1]);
- _dst3.val[1] = vreinterpret_u8_u32(_src37ttt_g.val[1]);
- _dst4.val[1] = vreinterpret_u8_u32(_src04ttt_g.val[0]);
- _dst5.val[1] = vreinterpret_u8_u32(_src15ttt_g.val[0]);
- _dst6.val[1] = vreinterpret_u8_u32(_src26ttt_g.val[0]);
- _dst7.val[1] = vreinterpret_u8_u32(_src37ttt_g.val[0]);
-
- _dst0.val[2] = vreinterpret_u8_u32(_src04ttt_b.val[1]);
- _dst1.val[2] = vreinterpret_u8_u32(_src15ttt_b.val[1]);
- _dst2.val[2] = vreinterpret_u8_u32(_src26ttt_b.val[1]);
- _dst3.val[2] = vreinterpret_u8_u32(_src37ttt_b.val[1]);
- _dst4.val[2] = vreinterpret_u8_u32(_src04ttt_b.val[0]);
- _dst5.val[2] = vreinterpret_u8_u32(_src15ttt_b.val[0]);
- _dst6.val[2] = vreinterpret_u8_u32(_src26ttt_b.val[0]);
- _dst7.val[2] = vreinterpret_u8_u32(_src37ttt_b.val[0]);
-
- vst3_u8(dst7, _dst7);
- vst3_u8(dst6, _dst6);
- vst3_u8(dst7 + dst_step, _dst5);
- vst3_u8(dst6 + dst_step, _dst4);
- vst3_u8(dst7 + 2 * dst_step, _dst3);
- vst3_u8(dst6 + 2 * dst_step, _dst2);
- vst3_u8(dst7 + 3 * dst_step, _dst1);
- vst3_u8(dst6 + 3 * dst_step, _dst0);
-
- src0 += 3 * 8;
- src1 += 3 * 8;
-
- dst7 += 4 * dst_step;
- dst6 += 4 * dst_step;
- }
- #else
- if (nn > 0)
- {
- asm volatile(
- "0: \n"
- "pld [%1, #192] \n"
- "vld3.u8 {d0-d2}, [%1], %10 \n"
-
- "pld [%2, #192] \n"
- "vld3.u8 {d4-d6}, [%2], %10 \n"
-
- "pld [%1, #192] \n"
- "vld3.u8 {d8-d10}, [%1], %10 \n"
-
- "vtrn.u8 q2, q0 \n" // _src01t_r
- "vtrn.u8 d6, d2 \n"
-
- "pld [%2, #192] \n"
- "vld3.u8 {d12-d14}, [%2], %10\n"
-
- "pld [%1, #192] \n"
- "vld3.u8 {d16-d18}, [%1], %10\n"
-
- "vtrn.u8 q6, q4 \n" // _src23t_r
- "vtrn.u8 d14, d10 \n"
-
- "pld [%2, #192] \n"
- "vld3.u8 {d20-d22}, [%2], %10\n"
-
- "pld [%1, #192] \n"
- "vld3.u8 {d24-d26}, [%1], %10\n"
-
- "vtrn.u8 q10, q8 \n" // _src45t_r
- "vtrn.u8 d22, d18 \n"
-
- "pld [%2, #192] \n"
- "vld3.u8 {d28-d30}, [%2], %10\n"
-
- "vtrn.u8 q14, q12 \n" // _src67t_r
- "vtrn.u8 d30, d26 \n"
-
- "sub %1, %1, %10, lsl #2 \n" // restore src0
-
- "vtrn.u16 q4, q0 \n" // _src02tt_r
- "vtrn.u16 d10, d2 \n"
-
- "sub %2, %2, %10, lsl #2 \n" // restore src1
-
- "vtrn.u16 q6, q2 \n" // _src13tt_r
- "vtrn.u16 d14, d6 \n"
-
- "add %1, #24 \n" // src0 += 24
-
- "vtrn.u16 q12, q8 \n" // _src46tt_r
- "vtrn.u16 d26, d18 \n"
-
- "add %2, #24 \n" // src1 += 24
-
- "vtrn.u16 q14, q10 \n" // _src57tt_r
- "vtrn.u16 d30, d22 \n"
-
- "vtrn.u32 q12, q4 \n" // _src26ttt_r
- "vtrn.u32 d26, d10 \n"
-
- "vtrn.u32 q14, q6 \n" // _src37ttt_r
- "vst3.u8 {d24-d26}, [%4], %11\n"
- "vtrn.u32 d30, d14 \n"
-
- "vtrn.u32 q8, q0 \n" // _src04ttt_r
- "vst3.u8 {d28-d30}, [%3], %11\n"
- "vtrn.u32 d18, d2 \n"
-
- "vtrn.u32 q10, q2 \n" // _src15ttt_r
- "vst3.u8 {d16-d18}, [%4], %11\n"
- "vtrn.u32 d22, d6 \n"
-
- "subs %0, #1 \n"
-
- "vst3.u8 {d8-d10}, [%4], %11 \n"
- "vst3.u8 {d20-d22}, [%3], %11\n"
- "vst3.u8 {d12-d14}, [%3], %11\n"
- "vst3.u8 {d0-d2}, [%4], %11 \n"
- "vst3.u8 {d4-d6}, [%3], %11 \n"
-
- "bne 0b \n"
- : "=r"(nn), // %0
- "=r"(src0), // %1
- "=r"(src1), // %2
- "=r"(dst7), // %3
- "=r"(dst6) // %4
- : "0"(nn),
- "1"(src0),
- "2"(src1),
- "3"(dst7),
- "4"(dst6),
- "r"(src_step), // %10
- "r"(dst_step) // %11
- : "cc", "memory", "q0", "q1", "q2", "q3", "q4", "q5", "q6", "q7", "q8", "q9", "q10", "q11", "q12", "q13", "q14", "q15");
- }
- #endif // __aarch64__
- for (; remain > 0; remain--)
- {
- dst7[0] = src1[0 + 3 * src_step];
- dst7[1] = src1[1 + 3 * src_step];
- dst7[2] = src1[2 + 3 * src_step];
- dst7[3] = src0[0 + 3 * src_step];
- dst7[4] = src0[1 + 3 * src_step];
- dst7[5] = src0[2 + 3 * src_step];
- dst7[6] = src1[0 + 2 * src_step];
- dst7[7] = src1[1 + 2 * src_step];
- dst7[8] = src1[2 + 2 * src_step];
- dst7[9] = src0[0 + 2 * src_step];
- dst7[10] = src0[1 + 2 * src_step];
- dst7[11] = src0[2 + 2 * src_step];
- dst7[12] = src1[0 + src_step];
- dst7[13] = src1[1 + src_step];
- dst7[14] = src1[2 + src_step];
- dst7[15] = src0[0 + src_step];
- dst7[16] = src0[1 + src_step];
- dst7[17] = src0[2 + src_step];
- dst7[18] = src1[0];
- dst7[19] = src1[1];
- dst7[20] = src1[2];
- dst7[21] = src0[0];
- dst7[22] = src0[1];
- dst7[23] = src0[2];
-
- src0 += 3;
- src1 += 3;
-
- dst7 -= stride;
- }
-
- src0 += srcwgap + 7 * srcstride;
- }
- #endif // __ARM_NEON
- for (; y < srch; y++)
- {
- unsigned char* dst0 = dstend - y * 3 - 3;
-
- int x = 0;
- for (; x < srcw; x++)
- {
- dst0[0] = src0[0];
- dst0[1] = src0[1];
- dst0[2] = src0[2];
-
- src0 += 3;
- dst0 -= stride;
- }
-
- src0 += srcwgap;
- }
- }
-
- static void kanna_rotate_7_c4(const unsigned char* src, int srcw, int srch, int srcstride, unsigned char* dst, int w, int h, int stride)
- {
- const int srcwgap = srcstride - srcw * 4;
-
- // point to the last dst pixel
- unsigned char* dstend = dst + stride * (h - 1) + w * 4;
-
- const unsigned char* src0 = src;
-
- int y = 0;
- #if __ARM_NEON
- for (; y + 7 < srch; y += 8)
- {
- const unsigned char* src1 = src0 + srcstride;
-
- unsigned char* dst6 = dstend - y * 4 - 8 * 4 - stride;
- unsigned char* dst7 = dstend - y * 4 - 8 * 4;
-
- int src_step = 2 * srcstride;
- int dst_step = -2 * stride;
-
- int nn = srcw >> 3;
- int remain = srcw - (nn << 3);
-
- #if !NCNN_GNU_INLINE_ASM || __aarch64__
- for (; nn > 0; nn--)
- {
- uint8x8x4_t _src0 = vld4_u8(src0);
- uint8x8x4_t _src1 = vld4_u8(src1);
-
- uint8x8x4_t _src2 = vld4_u8(src0 + src_step);
- uint8x8x4_t _src3 = vld4_u8(src1 + src_step);
-
- uint8x8x4_t _src4 = vld4_u8(src0 + 2 * src_step);
- uint8x8x4_t _src5 = vld4_u8(src1 + 2 * src_step);
-
- uint8x8x4_t _src6 = vld4_u8(src0 + 3 * src_step);
- uint8x8x4_t _src7 = vld4_u8(src1 + 3 * src_step);
-
- uint8x8x2_t _src01t_r = vtrn_u8(_src1.val[0], _src0.val[0]);
- uint8x8x2_t _src23t_r = vtrn_u8(_src3.val[0], _src2.val[0]);
- uint8x8x2_t _src45t_r = vtrn_u8(_src5.val[0], _src4.val[0]);
- uint8x8x2_t _src67t_r = vtrn_u8(_src7.val[0], _src6.val[0]);
-
- uint8x8x2_t _src01t_g = vtrn_u8(_src1.val[1], _src0.val[1]);
- uint8x8x2_t _src23t_g = vtrn_u8(_src3.val[1], _src2.val[1]);
- uint8x8x2_t _src45t_g = vtrn_u8(_src5.val[1], _src4.val[1]);
- uint8x8x2_t _src67t_g = vtrn_u8(_src7.val[1], _src6.val[1]);
-
- uint8x8x2_t _src01t_b = vtrn_u8(_src1.val[2], _src0.val[2]);
- uint8x8x2_t _src23t_b = vtrn_u8(_src3.val[2], _src2.val[2]);
- uint8x8x2_t _src45t_b = vtrn_u8(_src5.val[2], _src4.val[2]);
- uint8x8x2_t _src67t_b = vtrn_u8(_src7.val[2], _src6.val[2]);
-
- uint8x8x2_t _src01t_a = vtrn_u8(_src1.val[3], _src0.val[3]);
- uint8x8x2_t _src23t_a = vtrn_u8(_src3.val[3], _src2.val[3]);
- uint8x8x2_t _src45t_a = vtrn_u8(_src5.val[3], _src4.val[3]);
- uint8x8x2_t _src67t_a = vtrn_u8(_src7.val[3], _src6.val[3]);
-
- uint16x4x2_t _src02tt_r = vtrn_u16(vreinterpret_u16_u8(_src23t_r.val[1]), vreinterpret_u16_u8(_src01t_r.val[1]));
- uint16x4x2_t _src13tt_r = vtrn_u16(vreinterpret_u16_u8(_src23t_r.val[0]), vreinterpret_u16_u8(_src01t_r.val[0]));
- uint16x4x2_t _src46tt_r = vtrn_u16(vreinterpret_u16_u8(_src67t_r.val[1]), vreinterpret_u16_u8(_src45t_r.val[1]));
- uint16x4x2_t _src57tt_r = vtrn_u16(vreinterpret_u16_u8(_src67t_r.val[0]), vreinterpret_u16_u8(_src45t_r.val[0]));
-
- uint16x4x2_t _src02tt_g = vtrn_u16(vreinterpret_u16_u8(_src23t_g.val[1]), vreinterpret_u16_u8(_src01t_g.val[1]));
- uint16x4x2_t _src13tt_g = vtrn_u16(vreinterpret_u16_u8(_src23t_g.val[0]), vreinterpret_u16_u8(_src01t_g.val[0]));
- uint16x4x2_t _src46tt_g = vtrn_u16(vreinterpret_u16_u8(_src67t_g.val[1]), vreinterpret_u16_u8(_src45t_g.val[1]));
- uint16x4x2_t _src57tt_g = vtrn_u16(vreinterpret_u16_u8(_src67t_g.val[0]), vreinterpret_u16_u8(_src45t_g.val[0]));
-
- uint16x4x2_t _src02tt_b = vtrn_u16(vreinterpret_u16_u8(_src23t_b.val[1]), vreinterpret_u16_u8(_src01t_b.val[1]));
- uint16x4x2_t _src13tt_b = vtrn_u16(vreinterpret_u16_u8(_src23t_b.val[0]), vreinterpret_u16_u8(_src01t_b.val[0]));
- uint16x4x2_t _src46tt_b = vtrn_u16(vreinterpret_u16_u8(_src67t_b.val[1]), vreinterpret_u16_u8(_src45t_b.val[1]));
- uint16x4x2_t _src57tt_b = vtrn_u16(vreinterpret_u16_u8(_src67t_b.val[0]), vreinterpret_u16_u8(_src45t_b.val[0]));
-
- uint16x4x2_t _src02tt_a = vtrn_u16(vreinterpret_u16_u8(_src23t_a.val[1]), vreinterpret_u16_u8(_src01t_a.val[1]));
- uint16x4x2_t _src13tt_a = vtrn_u16(vreinterpret_u16_u8(_src23t_a.val[0]), vreinterpret_u16_u8(_src01t_a.val[0]));
- uint16x4x2_t _src46tt_a = vtrn_u16(vreinterpret_u16_u8(_src67t_a.val[1]), vreinterpret_u16_u8(_src45t_a.val[1]));
- uint16x4x2_t _src57tt_a = vtrn_u16(vreinterpret_u16_u8(_src67t_a.val[0]), vreinterpret_u16_u8(_src45t_a.val[0]));
-
- uint32x2x2_t _src04ttt_r = vtrn_u32(vreinterpret_u32_u16(_src46tt_r.val[1]), vreinterpret_u32_u16(_src02tt_r.val[1]));
- uint32x2x2_t _src15ttt_r = vtrn_u32(vreinterpret_u32_u16(_src57tt_r.val[1]), vreinterpret_u32_u16(_src13tt_r.val[1]));
- uint32x2x2_t _src26ttt_r = vtrn_u32(vreinterpret_u32_u16(_src46tt_r.val[0]), vreinterpret_u32_u16(_src02tt_r.val[0]));
- uint32x2x2_t _src37ttt_r = vtrn_u32(vreinterpret_u32_u16(_src57tt_r.val[0]), vreinterpret_u32_u16(_src13tt_r.val[0]));
-
- uint32x2x2_t _src04ttt_g = vtrn_u32(vreinterpret_u32_u16(_src46tt_g.val[1]), vreinterpret_u32_u16(_src02tt_g.val[1]));
- uint32x2x2_t _src15ttt_g = vtrn_u32(vreinterpret_u32_u16(_src57tt_g.val[1]), vreinterpret_u32_u16(_src13tt_g.val[1]));
- uint32x2x2_t _src26ttt_g = vtrn_u32(vreinterpret_u32_u16(_src46tt_g.val[0]), vreinterpret_u32_u16(_src02tt_g.val[0]));
- uint32x2x2_t _src37ttt_g = vtrn_u32(vreinterpret_u32_u16(_src57tt_g.val[0]), vreinterpret_u32_u16(_src13tt_g.val[0]));
-
- uint32x2x2_t _src04ttt_b = vtrn_u32(vreinterpret_u32_u16(_src46tt_b.val[1]), vreinterpret_u32_u16(_src02tt_b.val[1]));
- uint32x2x2_t _src15ttt_b = vtrn_u32(vreinterpret_u32_u16(_src57tt_b.val[1]), vreinterpret_u32_u16(_src13tt_b.val[1]));
- uint32x2x2_t _src26ttt_b = vtrn_u32(vreinterpret_u32_u16(_src46tt_b.val[0]), vreinterpret_u32_u16(_src02tt_b.val[0]));
- uint32x2x2_t _src37ttt_b = vtrn_u32(vreinterpret_u32_u16(_src57tt_b.val[0]), vreinterpret_u32_u16(_src13tt_b.val[0]));
-
- uint32x2x2_t _src04ttt_a = vtrn_u32(vreinterpret_u32_u16(_src46tt_a.val[1]), vreinterpret_u32_u16(_src02tt_a.val[1]));
- uint32x2x2_t _src15ttt_a = vtrn_u32(vreinterpret_u32_u16(_src57tt_a.val[1]), vreinterpret_u32_u16(_src13tt_a.val[1]));
- uint32x2x2_t _src26ttt_a = vtrn_u32(vreinterpret_u32_u16(_src46tt_a.val[0]), vreinterpret_u32_u16(_src02tt_a.val[0]));
- uint32x2x2_t _src37ttt_a = vtrn_u32(vreinterpret_u32_u16(_src57tt_a.val[0]), vreinterpret_u32_u16(_src13tt_a.val[0]));
-
- uint8x8x4_t _dst0;
- uint8x8x4_t _dst1;
- uint8x8x4_t _dst2;
- uint8x8x4_t _dst3;
- uint8x8x4_t _dst4;
- uint8x8x4_t _dst5;
- uint8x8x4_t _dst6;
- uint8x8x4_t _dst7;
-
- _dst0.val[0] = vreinterpret_u8_u32(_src04ttt_r.val[1]);
- _dst1.val[0] = vreinterpret_u8_u32(_src15ttt_r.val[1]);
- _dst2.val[0] = vreinterpret_u8_u32(_src26ttt_r.val[1]);
- _dst3.val[0] = vreinterpret_u8_u32(_src37ttt_r.val[1]);
- _dst4.val[0] = vreinterpret_u8_u32(_src04ttt_r.val[0]);
- _dst5.val[0] = vreinterpret_u8_u32(_src15ttt_r.val[0]);
- _dst6.val[0] = vreinterpret_u8_u32(_src26ttt_r.val[0]);
- _dst7.val[0] = vreinterpret_u8_u32(_src37ttt_r.val[0]);
-
- _dst0.val[1] = vreinterpret_u8_u32(_src04ttt_g.val[1]);
- _dst1.val[1] = vreinterpret_u8_u32(_src15ttt_g.val[1]);
- _dst2.val[1] = vreinterpret_u8_u32(_src26ttt_g.val[1]);
- _dst3.val[1] = vreinterpret_u8_u32(_src37ttt_g.val[1]);
- _dst4.val[1] = vreinterpret_u8_u32(_src04ttt_g.val[0]);
- _dst5.val[1] = vreinterpret_u8_u32(_src15ttt_g.val[0]);
- _dst6.val[1] = vreinterpret_u8_u32(_src26ttt_g.val[0]);
- _dst7.val[1] = vreinterpret_u8_u32(_src37ttt_g.val[0]);
-
- _dst0.val[2] = vreinterpret_u8_u32(_src04ttt_b.val[1]);
- _dst1.val[2] = vreinterpret_u8_u32(_src15ttt_b.val[1]);
- _dst2.val[2] = vreinterpret_u8_u32(_src26ttt_b.val[1]);
- _dst3.val[2] = vreinterpret_u8_u32(_src37ttt_b.val[1]);
- _dst4.val[2] = vreinterpret_u8_u32(_src04ttt_b.val[0]);
- _dst5.val[2] = vreinterpret_u8_u32(_src15ttt_b.val[0]);
- _dst6.val[2] = vreinterpret_u8_u32(_src26ttt_b.val[0]);
- _dst7.val[2] = vreinterpret_u8_u32(_src37ttt_b.val[0]);
-
- _dst0.val[3] = vreinterpret_u8_u32(_src04ttt_a.val[1]);
- _dst1.val[3] = vreinterpret_u8_u32(_src15ttt_a.val[1]);
- _dst2.val[3] = vreinterpret_u8_u32(_src26ttt_a.val[1]);
- _dst3.val[3] = vreinterpret_u8_u32(_src37ttt_a.val[1]);
- _dst4.val[3] = vreinterpret_u8_u32(_src04ttt_a.val[0]);
- _dst5.val[3] = vreinterpret_u8_u32(_src15ttt_a.val[0]);
- _dst6.val[3] = vreinterpret_u8_u32(_src26ttt_a.val[0]);
- _dst7.val[3] = vreinterpret_u8_u32(_src37ttt_a.val[0]);
-
- vst4_u8(dst7, _dst7);
- vst4_u8(dst6, _dst6);
- vst4_u8(dst7 + dst_step, _dst5);
- vst4_u8(dst6 + dst_step, _dst4);
- vst4_u8(dst7 + 2 * dst_step, _dst3);
- vst4_u8(dst6 + 2 * dst_step, _dst2);
- vst4_u8(dst7 + 3 * dst_step, _dst1);
- vst4_u8(dst6 + 3 * dst_step, _dst0);
-
- src0 += 4 * 8;
- src1 += 4 * 8;
-
- dst7 += 4 * dst_step;
- dst6 += 4 * dst_step;
- }
- #else
- if (nn > 0)
- {
- asm volatile(
- "0: \n"
- "pld [%1, #256] \n"
- "vld4.u8 {d0-d3}, [%1], %10 \n"
-
- "pld [%2, #256] \n"
- "vld4.u8 {d4-d7}, [%2], %10 \n"
-
- "pld [%1, #256] \n"
- "vld4.u8 {d8-d11}, [%1], %10 \n"
-
- "vtrn.u8 q2, q0 \n" // _src01t_r
- "vtrn.u8 q3, q1 \n"
-
- "pld [%2, #256] \n"
- "vld4.u8 {d12-d15}, [%2], %10\n"
-
- "pld [%1, #256] \n"
- "vld4.u8 {d16-d19}, [%1], %10\n"
-
- "vtrn.u8 q6, q4 \n" // _src23t_r
- "vtrn.u8 q7, q5 \n"
-
- "pld [%2, #256] \n"
- "vld4.u8 {d20-d23}, [%2], %10\n"
-
- "pld [%1, #256] \n"
- "vld4.u8 {d24-d27}, [%1], %10\n"
-
- "vtrn.u8 q10, q8 \n" // _src45t_r
- "vtrn.u8 q11, q9 \n"
-
- "pld [%2, #256] \n"
- "vld4.u8 {d28-d31}, [%2], %10\n"
-
- "vtrn.u8 q14, q12 \n" // _src67t_r
- "vtrn.u8 q15, q13 \n"
-
- "sub %1, %1, %10, lsl #2 \n" // restore src0
-
- "vtrn.u16 q4, q0 \n" // _src02tt_r
- "vtrn.u16 q5, q1 \n"
-
- "sub %2, %2, %10, lsl #2 \n" // restore src1
-
- "vtrn.u16 q6, q2 \n" // _src13tt_r
- "vtrn.u16 q7, q3 \n"
-
- "add %1, #32 \n" // src0 += 32
-
- "vtrn.u16 q12, q8 \n" // _src46tt_r
- "vtrn.u16 q13, q9 \n"
-
- "add %2, #32 \n" // src1 += 32
-
- "vtrn.u16 q14, q10 \n" // _src57tt_r
- "vtrn.u16 q15, q11 \n"
-
- "vtrn.u32 q12, q4 \n" // _src26ttt_r
- "vtrn.u32 q13, q5 \n"
-
- "vtrn.u32 q14, q6 \n" // _src37ttt_r
- "vst4.u8 {d24-d27}, [%4], %11\n"
- "vtrn.u32 q15, q7 \n"
-
- "vtrn.u32 q8, q0 \n" // _src04ttt_r
- "vst4.u8 {d28-d31}, [%3], %11\n"
- "vtrn.u32 q9, q1 \n"
-
- "vtrn.u32 q10, q2 \n" // _src15ttt_r
- "vst4.u8 {d16-d19}, [%4], %11\n"
- "vtrn.u32 q11, q3 \n"
-
- "subs %0, #1 \n"
-
- "vst4.u8 {d8-d11}, [%4], %11 \n"
- "vst4.u8 {d20-d23}, [%3], %11\n"
- "vst4.u8 {d12-d15}, [%3], %11\n"
- "vst4.u8 {d0-d3}, [%4], %11 \n"
- "vst4.u8 {d4-d7}, [%3], %11 \n"
-
- "bne 0b \n"
- : "=r"(nn), // %0
- "=r"(src0), // %1
- "=r"(src1), // %2
- "=r"(dst7), // %3
- "=r"(dst6) // %4
- : "0"(nn),
- "1"(src0),
- "2"(src1),
- "3"(dst7),
- "4"(dst6),
- "r"(src_step), // %10
- "r"(dst_step) // %11
- : "cc", "memory", "q0", "q1", "q2", "q3", "q4", "q5", "q6", "q7", "q8", "q9", "q10", "q11", "q12", "q13", "q14", "q15");
- }
- #endif // __aarch64__
- for (; remain > 0; remain--)
- {
- dst7[0] = src1[0 + 3 * src_step];
- dst7[1] = src1[1 + 3 * src_step];
- dst7[2] = src1[2 + 3 * src_step];
- dst7[3] = src1[3 + 3 * src_step];
- dst7[4] = src0[0 + 3 * src_step];
- dst7[5] = src0[1 + 3 * src_step];
- dst7[6] = src0[2 + 3 * src_step];
- dst7[7] = src0[3 + 3 * src_step];
- dst7[8] = src1[0 + 2 * src_step];
- dst7[9] = src1[1 + 2 * src_step];
- dst7[10] = src1[2 + 2 * src_step];
- dst7[11] = src1[3 + 2 * src_step];
- dst7[12] = src0[0 + 2 * src_step];
- dst7[13] = src0[1 + 2 * src_step];
- dst7[14] = src0[2 + 2 * src_step];
- dst7[15] = src0[3 + 2 * src_step];
- dst7[16] = src1[0 + src_step];
- dst7[17] = src1[1 + src_step];
- dst7[18] = src1[2 + src_step];
- dst7[19] = src1[3 + src_step];
- dst7[20] = src0[0 + src_step];
- dst7[21] = src0[1 + src_step];
- dst7[22] = src0[2 + src_step];
- dst7[23] = src0[3 + src_step];
- dst7[24] = src1[0];
- dst7[25] = src1[1];
- dst7[26] = src1[2];
- dst7[27] = src1[3];
- dst7[28] = src0[0];
- dst7[29] = src0[1];
- dst7[30] = src0[2];
- dst7[31] = src0[3];
-
- src0 += 4;
- src1 += 4;
-
- dst7 -= stride;
- }
-
- src0 += srcwgap + 7 * srcstride;
- }
- #endif // __ARM_NEON
- for (; y < srch; y++)
- {
- unsigned char* dst0 = dstend - y * 4 - 4;
-
- int x = 0;
- for (; x < srcw; x++)
- {
- dst0[0] = src0[0];
- dst0[1] = src0[1];
- dst0[2] = src0[2];
- dst0[3] = src0[3];
-
- src0 += 4;
- dst0 -= stride;
- }
-
- src0 += srcwgap;
- }
- }
-
- static void kanna_rotate_8_c1(const unsigned char* src, int srcw, int srch, int srcstride, unsigned char* dst, int /*w*/, int h, int stride)
- {
- const int srcwgap = srcstride - srcw;
-
- // point to the last dst pixel row
- unsigned char* dstend = dst + stride * (h - 1);
-
- const unsigned char* src0 = src;
-
- int y = 0;
- #if __ARM_NEON
- for (; y + 7 < srch; y += 8)
- {
- const unsigned char* src1 = src0 + srcstride;
-
- unsigned char* dst7 = dstend + y;
- unsigned char* dst6 = dstend + y - stride;
-
- int src_step = 2 * srcstride;
- int dst_step = -2 * stride;
-
- int nn = srcw >> 3;
- int remain = srcw - (nn << 3);
-
- #if !NCNN_GNU_INLINE_ASM || __aarch64__
- for (; nn > 0; nn--)
- {
- uint8x8_t _src0 = vld1_u8(src0);
- uint8x8_t _src1 = vld1_u8(src1);
-
- uint8x8_t _src2 = vld1_u8(src0 + src_step);
- uint8x8_t _src3 = vld1_u8(src1 + src_step);
-
- uint8x8_t _src4 = vld1_u8(src0 + 2 * src_step);
- uint8x8_t _src5 = vld1_u8(src1 + 2 * src_step);
-
- uint8x8_t _src6 = vld1_u8(src0 + 3 * src_step);
- uint8x8_t _src7 = vld1_u8(src1 + 3 * src_step);
-
- uint8x8x2_t _src01t_r = vtrn_u8(_src0, _src1);
- uint8x8x2_t _src23t_r = vtrn_u8(_src2, _src3);
- uint8x8x2_t _src45t_r = vtrn_u8(_src4, _src5);
- uint8x8x2_t _src67t_r = vtrn_u8(_src6, _src7);
-
- uint16x4x2_t _src02tt_r = vtrn_u16(vreinterpret_u16_u8(_src01t_r.val[0]), vreinterpret_u16_u8(_src23t_r.val[0]));
- uint16x4x2_t _src13tt_r = vtrn_u16(vreinterpret_u16_u8(_src01t_r.val[1]), vreinterpret_u16_u8(_src23t_r.val[1]));
- uint16x4x2_t _src46tt_r = vtrn_u16(vreinterpret_u16_u8(_src45t_r.val[0]), vreinterpret_u16_u8(_src67t_r.val[0]));
- uint16x4x2_t _src57tt_r = vtrn_u16(vreinterpret_u16_u8(_src45t_r.val[1]), vreinterpret_u16_u8(_src67t_r.val[1]));
-
- uint32x2x2_t _src04ttt_r = vtrn_u32(vreinterpret_u32_u16(_src02tt_r.val[0]), vreinterpret_u32_u16(_src46tt_r.val[0]));
- uint32x2x2_t _src15ttt_r = vtrn_u32(vreinterpret_u32_u16(_src13tt_r.val[0]), vreinterpret_u32_u16(_src57tt_r.val[0]));
- uint32x2x2_t _src26ttt_r = vtrn_u32(vreinterpret_u32_u16(_src02tt_r.val[1]), vreinterpret_u32_u16(_src46tt_r.val[1]));
- uint32x2x2_t _src37ttt_r = vtrn_u32(vreinterpret_u32_u16(_src13tt_r.val[1]), vreinterpret_u32_u16(_src57tt_r.val[1]));
-
- uint8x8_t _dst0 = vreinterpret_u8_u32(_src04ttt_r.val[0]);
- uint8x8_t _dst1 = vreinterpret_u8_u32(_src15ttt_r.val[0]);
- uint8x8_t _dst2 = vreinterpret_u8_u32(_src26ttt_r.val[0]);
- uint8x8_t _dst3 = vreinterpret_u8_u32(_src37ttt_r.val[0]);
- uint8x8_t _dst4 = vreinterpret_u8_u32(_src04ttt_r.val[1]);
- uint8x8_t _dst5 = vreinterpret_u8_u32(_src15ttt_r.val[1]);
- uint8x8_t _dst6 = vreinterpret_u8_u32(_src26ttt_r.val[1]);
- uint8x8_t _dst7 = vreinterpret_u8_u32(_src37ttt_r.val[1]);
-
- vst1_u8(dst7, _dst0);
- vst1_u8(dst6, _dst1);
- vst1_u8(dst7 + dst_step, _dst2);
- vst1_u8(dst6 + dst_step, _dst3);
- vst1_u8(dst7 + 2 * dst_step, _dst4);
- vst1_u8(dst6 + 2 * dst_step, _dst5);
- vst1_u8(dst7 + 3 * dst_step, _dst6);
- vst1_u8(dst6 + 3 * dst_step, _dst7);
-
- src0 += 8;
- src1 += 8;
-
- dst7 += 4 * dst_step;
- dst6 += 4 * dst_step;
- }
- #else
- if (nn > 0)
- {
- asm volatile(
- "0: \n"
- "pld [%1, #64] \n"
- "vld1.u8 {d0}, [%1], %10 \n"
-
- "pld [%2, #64] \n"
- "vld1.u8 {d1}, [%2], %10 \n"
-
- "pld [%1, #64] \n"
- "vld1.u8 {d2}, [%1], %10 \n"
-
- "vtrn.u8 d0, d1 \n" // _src01t_r
-
- "pld [%2, #64] \n"
- "vld1.u8 {d3}, [%2], %10 \n"
-
- "pld [%1, #64] \n"
- "vld1.u8 {d4}, [%1], %10 \n"
-
- "vtrn.u8 d2, d3 \n" // _src23t_r
-
- "pld [%2, #64] \n"
- "vld1.u8 {d5}, [%2], %10 \n"
-
- "pld [%1, #64] \n"
- "vld1.u8 {d6}, [%1], %10 \n"
-
- "vtrn.u8 d4, d5 \n" // _src45t_r
-
- "pld [%2, #64] \n"
- "vld1.u8 {d7}, [%2], %10 \n"
-
- "vtrn.u8 d6, d7 \n" // _src67t_r
-
- "sub %1, %1, %10, lsl #2 \n" // restore src0
-
- "vtrn.u16 q0, q1 \n" // _src02tt_r _src13tt_r
-
- "sub %2, %2, %10, lsl #2 \n" // restore src1
-
- "vtrn.u16 q2, q3 \n" // _src46tt_r _src57tt_r
-
- "add %1, #8 \n" // src0 += 8
-
- "vtrn.u32 q0, q2 \n" // _src04ttt_r _src15ttt_r
-
- "add %2, #8 \n" // src1 += 8
-
- "vtrn.u32 q1, q3 \n" // _src26ttt_r _src37ttt_r
- "vst1.u8 {d0}, [%3], %11 \n"
- "vst1.u8 {d1}, [%4], %11 \n"
-
- "subs %0, #1 \n"
-
- "vst1.u8 {d2}, [%3], %11 \n"
- "vst1.u8 {d3}, [%4], %11 \n"
- "vst1.u8 {d4}, [%3], %11 \n"
- "vst1.u8 {d5}, [%4], %11 \n"
- "vst1.u8 {d6}, [%3], %11 \n"
- "vst1.u8 {d7}, [%4], %11 \n"
-
- "bne 0b \n"
- : "=r"(nn), // %0
- "=r"(src0), // %1
- "=r"(src1), // %2
- "=r"(dst7), // %3
- "=r"(dst6) // %4
- : "0"(nn),
- "1"(src0),
- "2"(src1),
- "3"(dst7),
- "4"(dst6),
- "r"(src_step), // %10
- "r"(dst_step) // %11
- : "cc", "memory", "q0", "q1", "q2", "q3");
- }
- #endif // __aarch64__
- for (; remain > 0; remain--)
- {
- dst7[0] = src0[0];
- dst7[1] = src1[0];
- dst7[2] = src0[0 + src_step];
- dst7[3] = src1[0 + src_step];
- dst7[4] = src0[0 + 2 * src_step];
- dst7[5] = src1[0 + 2 * src_step];
- dst7[6] = src0[0 + 3 * src_step];
- dst7[7] = src1[0 + 3 * src_step];
-
- src0 += 1;
- src1 += 1;
-
- dst7 -= stride;
- }
-
- src0 += srcwgap + 7 * srcstride;
- }
- #endif // __ARM_NEON
- for (; y < srch; y++)
- {
- unsigned char* dst0 = dstend + y;
-
- int x = 0;
- for (; x < srcw; x++)
- {
- *dst0 = *src0;
-
- src0 += 1;
- dst0 -= stride;
- }
-
- src0 += srcwgap;
- }
- }
-
- static void kanna_rotate_8_c2(const unsigned char* src, int srcw, int srch, int srcstride, unsigned char* dst, int /*w*/, int h, int stride)
- {
- const int srcwgap = srcstride - srcw * 2;
-
- // point to the last dst pixel row
- unsigned char* dstend = dst + stride * (h - 1);
-
- const unsigned char* src0 = src;
-
- int y = 0;
- #if __ARM_NEON
- for (; y + 7 < srch; y += 8)
- {
- const unsigned char* src1 = src0 + srcstride;
-
- unsigned char* dst7 = dstend + y * 2;
- unsigned char* dst6 = dstend + y * 2 - stride;
-
- int src_step = 2 * srcstride;
- int dst_step = -2 * stride;
-
- int nn = srcw >> 3;
- int remain = srcw - (nn << 3);
-
- #if !NCNN_GNU_INLINE_ASM || __aarch64__
- for (; nn > 0; nn--)
- {
- uint8x8x2_t _src0 = vld2_u8(src0);
- uint8x8x2_t _src1 = vld2_u8(src1);
-
- uint8x8x2_t _src2 = vld2_u8(src0 + src_step);
- uint8x8x2_t _src3 = vld2_u8(src1 + src_step);
-
- uint8x8x2_t _src4 = vld2_u8(src0 + 2 * src_step);
- uint8x8x2_t _src5 = vld2_u8(src1 + 2 * src_step);
-
- uint8x8x2_t _src6 = vld2_u8(src0 + 3 * src_step);
- uint8x8x2_t _src7 = vld2_u8(src1 + 3 * src_step);
-
- uint8x8x2_t _src01t_r = vtrn_u8(_src0.val[0], _src1.val[0]);
- uint8x8x2_t _src23t_r = vtrn_u8(_src2.val[0], _src3.val[0]);
- uint8x8x2_t _src45t_r = vtrn_u8(_src4.val[0], _src5.val[0]);
- uint8x8x2_t _src67t_r = vtrn_u8(_src6.val[0], _src7.val[0]);
-
- uint8x8x2_t _src01t_g = vtrn_u8(_src0.val[1], _src1.val[1]);
- uint8x8x2_t _src23t_g = vtrn_u8(_src2.val[1], _src3.val[1]);
- uint8x8x2_t _src45t_g = vtrn_u8(_src4.val[1], _src5.val[1]);
- uint8x8x2_t _src67t_g = vtrn_u8(_src6.val[1], _src7.val[1]);
-
- uint16x4x2_t _src02tt_r = vtrn_u16(vreinterpret_u16_u8(_src01t_r.val[0]), vreinterpret_u16_u8(_src23t_r.val[0]));
- uint16x4x2_t _src13tt_r = vtrn_u16(vreinterpret_u16_u8(_src01t_r.val[1]), vreinterpret_u16_u8(_src23t_r.val[1]));
- uint16x4x2_t _src46tt_r = vtrn_u16(vreinterpret_u16_u8(_src45t_r.val[0]), vreinterpret_u16_u8(_src67t_r.val[0]));
- uint16x4x2_t _src57tt_r = vtrn_u16(vreinterpret_u16_u8(_src45t_r.val[1]), vreinterpret_u16_u8(_src67t_r.val[1]));
-
- uint16x4x2_t _src02tt_g = vtrn_u16(vreinterpret_u16_u8(_src01t_g.val[0]), vreinterpret_u16_u8(_src23t_g.val[0]));
- uint16x4x2_t _src13tt_g = vtrn_u16(vreinterpret_u16_u8(_src01t_g.val[1]), vreinterpret_u16_u8(_src23t_g.val[1]));
- uint16x4x2_t _src46tt_g = vtrn_u16(vreinterpret_u16_u8(_src45t_g.val[0]), vreinterpret_u16_u8(_src67t_g.val[0]));
- uint16x4x2_t _src57tt_g = vtrn_u16(vreinterpret_u16_u8(_src45t_g.val[1]), vreinterpret_u16_u8(_src67t_g.val[1]));
-
- uint32x2x2_t _src04ttt_r = vtrn_u32(vreinterpret_u32_u16(_src02tt_r.val[0]), vreinterpret_u32_u16(_src46tt_r.val[0]));
- uint32x2x2_t _src15ttt_r = vtrn_u32(vreinterpret_u32_u16(_src13tt_r.val[0]), vreinterpret_u32_u16(_src57tt_r.val[0]));
- uint32x2x2_t _src26ttt_r = vtrn_u32(vreinterpret_u32_u16(_src02tt_r.val[1]), vreinterpret_u32_u16(_src46tt_r.val[1]));
- uint32x2x2_t _src37ttt_r = vtrn_u32(vreinterpret_u32_u16(_src13tt_r.val[1]), vreinterpret_u32_u16(_src57tt_r.val[1]));
-
- uint32x2x2_t _src04ttt_g = vtrn_u32(vreinterpret_u32_u16(_src02tt_g.val[0]), vreinterpret_u32_u16(_src46tt_g.val[0]));
- uint32x2x2_t _src15ttt_g = vtrn_u32(vreinterpret_u32_u16(_src13tt_g.val[0]), vreinterpret_u32_u16(_src57tt_g.val[0]));
- uint32x2x2_t _src26ttt_g = vtrn_u32(vreinterpret_u32_u16(_src02tt_g.val[1]), vreinterpret_u32_u16(_src46tt_g.val[1]));
- uint32x2x2_t _src37ttt_g = vtrn_u32(vreinterpret_u32_u16(_src13tt_g.val[1]), vreinterpret_u32_u16(_src57tt_g.val[1]));
-
- uint8x8x2_t _dst0;
- uint8x8x2_t _dst1;
- uint8x8x2_t _dst2;
- uint8x8x2_t _dst3;
- uint8x8x2_t _dst4;
- uint8x8x2_t _dst5;
- uint8x8x2_t _dst6;
- uint8x8x2_t _dst7;
-
- _dst0.val[0] = vreinterpret_u8_u32(_src04ttt_r.val[0]);
- _dst1.val[0] = vreinterpret_u8_u32(_src15ttt_r.val[0]);
- _dst2.val[0] = vreinterpret_u8_u32(_src26ttt_r.val[0]);
- _dst3.val[0] = vreinterpret_u8_u32(_src37ttt_r.val[0]);
- _dst4.val[0] = vreinterpret_u8_u32(_src04ttt_r.val[1]);
- _dst5.val[0] = vreinterpret_u8_u32(_src15ttt_r.val[1]);
- _dst6.val[0] = vreinterpret_u8_u32(_src26ttt_r.val[1]);
- _dst7.val[0] = vreinterpret_u8_u32(_src37ttt_r.val[1]);
-
- _dst0.val[1] = vreinterpret_u8_u32(_src04ttt_g.val[0]);
- _dst1.val[1] = vreinterpret_u8_u32(_src15ttt_g.val[0]);
- _dst2.val[1] = vreinterpret_u8_u32(_src26ttt_g.val[0]);
- _dst3.val[1] = vreinterpret_u8_u32(_src37ttt_g.val[0]);
- _dst4.val[1] = vreinterpret_u8_u32(_src04ttt_g.val[1]);
- _dst5.val[1] = vreinterpret_u8_u32(_src15ttt_g.val[1]);
- _dst6.val[1] = vreinterpret_u8_u32(_src26ttt_g.val[1]);
- _dst7.val[1] = vreinterpret_u8_u32(_src37ttt_g.val[1]);
-
- vst2_u8(dst7, _dst0);
- vst2_u8(dst6, _dst1);
- vst2_u8(dst7 + dst_step, _dst2);
- vst2_u8(dst6 + dst_step, _dst3);
- vst2_u8(dst7 + 2 * dst_step, _dst4);
- vst2_u8(dst6 + 2 * dst_step, _dst5);
- vst2_u8(dst7 + 3 * dst_step, _dst6);
- vst2_u8(dst6 + 3 * dst_step, _dst7);
-
- src0 += 2 * 8;
- src1 += 2 * 8;
-
- dst7 += 4 * dst_step;
- dst6 += 4 * dst_step;
- }
- #else
- if (nn > 0)
- {
- asm volatile(
- "0: \n"
- "pld [%1, #128] \n"
- "vld2.u8 {d0-d1}, [%1], %10 \n"
-
- "pld [%2, #128] \n"
- "vld2.u8 {d2-d3}, [%2], %10 \n"
-
- "pld [%1, #128] \n"
- "vld2.u8 {d4-d5}, [%1], %10 \n"
-
- "vtrn.u8 q0, q1 \n" // _src01t_r
-
- "pld [%2, #128] \n"
- "vld2.u8 {d6-d7}, [%2], %10 \n"
-
- "pld [%1, #128] \n"
- "vld2.u8 {d16-d17}, [%1], %10\n"
-
- "vtrn.u8 q2, q3 \n" // _src23t_r
-
- "pld [%2, #128] \n"
- "vld2.u8 {d18-d19}, [%2], %10\n"
-
- "pld [%1, #128] \n"
- "vld2.u8 {d20-d21}, [%1], %10\n"
-
- "vtrn.u8 q8, q9 \n" // _src45t_r
-
- "pld [%2, #128] \n"
- "vld2.u8 {d22-d23}, [%2], %10\n"
-
- "vtrn.u8 q10, q11 \n" // _src67t_r
-
- "sub %1, %1, %10, lsl #2 \n" // restore src0
-
- "vtrn.u16 q0, q2 \n" // _src02tt_r
-
- "sub %2, %2, %10, lsl #2 \n" // restore src1
-
- "vtrn.u16 q1, q3 \n" // _src13tt_r
-
- "add %1, #16 \n" // src0 += 16
-
- "vtrn.u16 q8, q10 \n" // _src46tt_r
-
- "add %2, #16 \n" // src1 += 16
-
- "vtrn.u16 q9, q11 \n" // _src57tt_r
-
- "vtrn.u32 q0, q8 \n" // _src04ttt_r
-
- "vtrn.u32 q1, q9 \n" // _src15ttt_r
- "vst2.u8 {d0-d1}, [%3], %11 \n"
-
- "vtrn.u32 q2, q10 \n" // _src26ttt_r
- "vst2.u8 {d2-d3}, [%4], %11 \n"
-
- "vtrn.u32 q3, q11 \n" // _src37ttt_r
- "vst2.u8 {d4-d5}, [%3], %11 \n"
-
- "subs %0, #1 \n"
-
- "vst2.u8 {d16-d17}, [%3], %11\n"
- "vst2.u8 {d6-d7}, [%4], %11 \n"
- "vst2.u8 {d18-d19}, [%4], %11\n"
- "vst2.u8 {d20-d21}, [%3], %11\n"
- "vst2.u8 {d22-d23}, [%4], %11\n"
-
- "bne 0b \n"
- : "=r"(nn), // %0
- "=r"(src0), // %1
- "=r"(src1), // %2
- "=r"(dst7), // %3
- "=r"(dst6) // %4
- : "0"(nn),
- "1"(src0),
- "2"(src1),
- "3"(dst7),
- "4"(dst6),
- "r"(src_step), // %10
- "r"(dst_step) // %11
- : "cc", "memory", "q0", "q1", "q2", "q3", "q8", "q9", "q10", "q11");
- }
- #endif // __aarch64__
- for (; remain > 0; remain--)
- {
- dst7[0] = src0[0];
- dst7[1] = src0[1];
- dst7[2] = src1[0];
- dst7[3] = src1[1];
- dst7[4] = src0[0 + src_step];
- dst7[5] = src0[1 + src_step];
- dst7[6] = src1[0 + src_step];
- dst7[7] = src1[1 + src_step];
- dst7[8] = src0[0 + 2 * src_step];
- dst7[9] = src0[1 + 2 * src_step];
- dst7[10] = src1[0 + 2 * src_step];
- dst7[11] = src1[1 + 2 * src_step];
- dst7[12] = src0[0 + 3 * src_step];
- dst7[13] = src0[1 + 3 * src_step];
- dst7[14] = src1[0 + 3 * src_step];
- dst7[15] = src1[1 + 3 * src_step];
-
- src0 += 2;
- src1 += 2;
-
- dst7 -= stride;
- }
-
- src0 += srcwgap + 7 * srcstride;
- }
- #endif // __ARM_NEON
- for (; y < srch; y++)
- {
- unsigned char* dst0 = dstend + y * 2;
-
- int x = 0;
- for (; x < srcw; x++)
- {
- dst0[0] = src0[0];
- dst0[1] = src0[1];
-
- src0 += 2;
- dst0 -= stride;
- }
-
- src0 += srcwgap;
- }
- }
-
- static void kanna_rotate_8_c3(const unsigned char* src, int srcw, int srch, int srcstride, unsigned char* dst, int /*w*/, int h, int stride)
- {
- const int srcwgap = srcstride - srcw * 3;
-
- // point to the last dst pixel row
- unsigned char* dstend = dst + stride * (h - 1);
-
- const unsigned char* src0 = src;
-
- int y = 0;
- #if __ARM_NEON
- for (; y + 7 < srch; y += 8)
- {
- const unsigned char* src1 = src0 + srcstride;
-
- unsigned char* dst7 = dstend + y * 3;
- unsigned char* dst6 = dstend + y * 3 - stride;
-
- int src_step = 2 * srcstride;
- int dst_step = -2 * stride;
-
- int nn = srcw >> 3;
- int remain = srcw - (nn << 3);
-
- #if !NCNN_GNU_INLINE_ASM || __aarch64__
- for (; nn > 0; nn--)
- {
- uint8x8x3_t _src0 = vld3_u8(src0);
- uint8x8x3_t _src1 = vld3_u8(src1);
-
- uint8x8x3_t _src2 = vld3_u8(src0 + src_step);
- uint8x8x3_t _src3 = vld3_u8(src1 + src_step);
-
- uint8x8x3_t _src4 = vld3_u8(src0 + 2 * src_step);
- uint8x8x3_t _src5 = vld3_u8(src1 + 2 * src_step);
-
- uint8x8x3_t _src6 = vld3_u8(src0 + 3 * src_step);
- uint8x8x3_t _src7 = vld3_u8(src1 + 3 * src_step);
-
- uint8x8x2_t _src01t_r = vtrn_u8(_src0.val[0], _src1.val[0]);
- uint8x8x2_t _src23t_r = vtrn_u8(_src2.val[0], _src3.val[0]);
- uint8x8x2_t _src45t_r = vtrn_u8(_src4.val[0], _src5.val[0]);
- uint8x8x2_t _src67t_r = vtrn_u8(_src6.val[0], _src7.val[0]);
-
- uint8x8x2_t _src01t_g = vtrn_u8(_src0.val[1], _src1.val[1]);
- uint8x8x2_t _src23t_g = vtrn_u8(_src2.val[1], _src3.val[1]);
- uint8x8x2_t _src45t_g = vtrn_u8(_src4.val[1], _src5.val[1]);
- uint8x8x2_t _src67t_g = vtrn_u8(_src6.val[1], _src7.val[1]);
-
- uint8x8x2_t _src01t_b = vtrn_u8(_src0.val[2], _src1.val[2]);
- uint8x8x2_t _src23t_b = vtrn_u8(_src2.val[2], _src3.val[2]);
- uint8x8x2_t _src45t_b = vtrn_u8(_src4.val[2], _src5.val[2]);
- uint8x8x2_t _src67t_b = vtrn_u8(_src6.val[2], _src7.val[2]);
-
- uint16x4x2_t _src02tt_r = vtrn_u16(vreinterpret_u16_u8(_src01t_r.val[0]), vreinterpret_u16_u8(_src23t_r.val[0]));
- uint16x4x2_t _src13tt_r = vtrn_u16(vreinterpret_u16_u8(_src01t_r.val[1]), vreinterpret_u16_u8(_src23t_r.val[1]));
- uint16x4x2_t _src46tt_r = vtrn_u16(vreinterpret_u16_u8(_src45t_r.val[0]), vreinterpret_u16_u8(_src67t_r.val[0]));
- uint16x4x2_t _src57tt_r = vtrn_u16(vreinterpret_u16_u8(_src45t_r.val[1]), vreinterpret_u16_u8(_src67t_r.val[1]));
-
- uint16x4x2_t _src02tt_g = vtrn_u16(vreinterpret_u16_u8(_src01t_g.val[0]), vreinterpret_u16_u8(_src23t_g.val[0]));
- uint16x4x2_t _src13tt_g = vtrn_u16(vreinterpret_u16_u8(_src01t_g.val[1]), vreinterpret_u16_u8(_src23t_g.val[1]));
- uint16x4x2_t _src46tt_g = vtrn_u16(vreinterpret_u16_u8(_src45t_g.val[0]), vreinterpret_u16_u8(_src67t_g.val[0]));
- uint16x4x2_t _src57tt_g = vtrn_u16(vreinterpret_u16_u8(_src45t_g.val[1]), vreinterpret_u16_u8(_src67t_g.val[1]));
-
- uint16x4x2_t _src02tt_b = vtrn_u16(vreinterpret_u16_u8(_src01t_b.val[0]), vreinterpret_u16_u8(_src23t_b.val[0]));
- uint16x4x2_t _src13tt_b = vtrn_u16(vreinterpret_u16_u8(_src01t_b.val[1]), vreinterpret_u16_u8(_src23t_b.val[1]));
- uint16x4x2_t _src46tt_b = vtrn_u16(vreinterpret_u16_u8(_src45t_b.val[0]), vreinterpret_u16_u8(_src67t_b.val[0]));
- uint16x4x2_t _src57tt_b = vtrn_u16(vreinterpret_u16_u8(_src45t_b.val[1]), vreinterpret_u16_u8(_src67t_b.val[1]));
-
- uint32x2x2_t _src04ttt_r = vtrn_u32(vreinterpret_u32_u16(_src02tt_r.val[0]), vreinterpret_u32_u16(_src46tt_r.val[0]));
- uint32x2x2_t _src15ttt_r = vtrn_u32(vreinterpret_u32_u16(_src13tt_r.val[0]), vreinterpret_u32_u16(_src57tt_r.val[0]));
- uint32x2x2_t _src26ttt_r = vtrn_u32(vreinterpret_u32_u16(_src02tt_r.val[1]), vreinterpret_u32_u16(_src46tt_r.val[1]));
- uint32x2x2_t _src37ttt_r = vtrn_u32(vreinterpret_u32_u16(_src13tt_r.val[1]), vreinterpret_u32_u16(_src57tt_r.val[1]));
-
- uint32x2x2_t _src04ttt_g = vtrn_u32(vreinterpret_u32_u16(_src02tt_g.val[0]), vreinterpret_u32_u16(_src46tt_g.val[0]));
- uint32x2x2_t _src15ttt_g = vtrn_u32(vreinterpret_u32_u16(_src13tt_g.val[0]), vreinterpret_u32_u16(_src57tt_g.val[0]));
- uint32x2x2_t _src26ttt_g = vtrn_u32(vreinterpret_u32_u16(_src02tt_g.val[1]), vreinterpret_u32_u16(_src46tt_g.val[1]));
- uint32x2x2_t _src37ttt_g = vtrn_u32(vreinterpret_u32_u16(_src13tt_g.val[1]), vreinterpret_u32_u16(_src57tt_g.val[1]));
-
- uint32x2x2_t _src04ttt_b = vtrn_u32(vreinterpret_u32_u16(_src02tt_b.val[0]), vreinterpret_u32_u16(_src46tt_b.val[0]));
- uint32x2x2_t _src15ttt_b = vtrn_u32(vreinterpret_u32_u16(_src13tt_b.val[0]), vreinterpret_u32_u16(_src57tt_b.val[0]));
- uint32x2x2_t _src26ttt_b = vtrn_u32(vreinterpret_u32_u16(_src02tt_b.val[1]), vreinterpret_u32_u16(_src46tt_b.val[1]));
- uint32x2x2_t _src37ttt_b = vtrn_u32(vreinterpret_u32_u16(_src13tt_b.val[1]), vreinterpret_u32_u16(_src57tt_b.val[1]));
-
- uint8x8x3_t _dst0;
- uint8x8x3_t _dst1;
- uint8x8x3_t _dst2;
- uint8x8x3_t _dst3;
- uint8x8x3_t _dst4;
- uint8x8x3_t _dst5;
- uint8x8x3_t _dst6;
- uint8x8x3_t _dst7;
-
- _dst0.val[0] = vreinterpret_u8_u32(_src04ttt_r.val[0]);
- _dst1.val[0] = vreinterpret_u8_u32(_src15ttt_r.val[0]);
- _dst2.val[0] = vreinterpret_u8_u32(_src26ttt_r.val[0]);
- _dst3.val[0] = vreinterpret_u8_u32(_src37ttt_r.val[0]);
- _dst4.val[0] = vreinterpret_u8_u32(_src04ttt_r.val[1]);
- _dst5.val[0] = vreinterpret_u8_u32(_src15ttt_r.val[1]);
- _dst6.val[0] = vreinterpret_u8_u32(_src26ttt_r.val[1]);
- _dst7.val[0] = vreinterpret_u8_u32(_src37ttt_r.val[1]);
-
- _dst0.val[1] = vreinterpret_u8_u32(_src04ttt_g.val[0]);
- _dst1.val[1] = vreinterpret_u8_u32(_src15ttt_g.val[0]);
- _dst2.val[1] = vreinterpret_u8_u32(_src26ttt_g.val[0]);
- _dst3.val[1] = vreinterpret_u8_u32(_src37ttt_g.val[0]);
- _dst4.val[1] = vreinterpret_u8_u32(_src04ttt_g.val[1]);
- _dst5.val[1] = vreinterpret_u8_u32(_src15ttt_g.val[1]);
- _dst6.val[1] = vreinterpret_u8_u32(_src26ttt_g.val[1]);
- _dst7.val[1] = vreinterpret_u8_u32(_src37ttt_g.val[1]);
-
- _dst0.val[2] = vreinterpret_u8_u32(_src04ttt_b.val[0]);
- _dst1.val[2] = vreinterpret_u8_u32(_src15ttt_b.val[0]);
- _dst2.val[2] = vreinterpret_u8_u32(_src26ttt_b.val[0]);
- _dst3.val[2] = vreinterpret_u8_u32(_src37ttt_b.val[0]);
- _dst4.val[2] = vreinterpret_u8_u32(_src04ttt_b.val[1]);
- _dst5.val[2] = vreinterpret_u8_u32(_src15ttt_b.val[1]);
- _dst6.val[2] = vreinterpret_u8_u32(_src26ttt_b.val[1]);
- _dst7.val[2] = vreinterpret_u8_u32(_src37ttt_b.val[1]);
-
- vst3_u8(dst7, _dst0);
- vst3_u8(dst6, _dst1);
- vst3_u8(dst7 + dst_step, _dst2);
- vst3_u8(dst6 + dst_step, _dst3);
- vst3_u8(dst7 + 2 * dst_step, _dst4);
- vst3_u8(dst6 + 2 * dst_step, _dst5);
- vst3_u8(dst7 + 3 * dst_step, _dst6);
- vst3_u8(dst6 + 3 * dst_step, _dst7);
-
- src0 += 3 * 8;
- src1 += 3 * 8;
-
- dst7 += 4 * dst_step;
- dst6 += 4 * dst_step;
- }
- #else
- if (nn > 0)
- {
- asm volatile(
- "0: \n"
- "pld [%1, #192] \n"
- "vld3.u8 {d0-d2}, [%1], %10 \n"
-
- "pld [%2, #192] \n"
- "vld3.u8 {d4-d6}, [%2], %10 \n"
-
- "pld [%1, #192] \n"
- "vld3.u8 {d8-d10}, [%1], %10 \n"
-
- "vtrn.u8 q0, q2 \n" // _src01t_r
- "vtrn.u8 d2, d6 \n"
-
- "pld [%2, #192] \n"
- "vld3.u8 {d12-d14}, [%2], %10\n"
-
- "pld [%1, #192] \n"
- "vld3.u8 {d16-d18}, [%1], %10\n"
-
- "vtrn.u8 q4, q6 \n" // _src23t_r
- "vtrn.u8 d10, d14 \n"
-
- "pld [%2, #192] \n"
- "vld3.u8 {d20-d22}, [%2], %10\n"
-
- "pld [%1, #192] \n"
- "vld3.u8 {d24-d26}, [%1], %10\n"
-
- "vtrn.u8 q8, q10 \n" // _src45t_r
- "vtrn.u8 d18, d22 \n"
-
- "pld [%2, #192] \n"
- "vld3.u8 {d28-d30}, [%2], %10\n"
-
- "vtrn.u8 q12, q14 \n" // _src67t_r
- "vtrn.u8 d26, d30 \n"
-
- "sub %1, %1, %10, lsl #2 \n" // restore src0
-
- "vtrn.u16 q0, q4 \n" // _src02tt_r
- "vtrn.u16 d2, d10 \n"
-
- "sub %2, %2, %10, lsl #2 \n" // restore src1
-
- "vtrn.u16 q2, q6 \n" // _src13tt_r
- "vtrn.u16 d6, d14 \n"
-
- "add %1, #24 \n" // src0 += 24
-
- "vtrn.u16 q8, q12 \n" // _src46tt_r
- "vtrn.u16 d18, d26 \n"
-
- "add %2, #24 \n" // src1 += 24
-
- "vtrn.u16 q10, q14 \n" // _src57tt_r
- "vtrn.u16 d22, d30 \n"
-
- "vtrn.u32 q0, q8 \n" // _src04ttt_r
- "vtrn.u32 d2, d18 \n"
-
- "vtrn.u32 q2, q10 \n" // _src15ttt_r
- "vst3.u8 {d0-d2}, [%3], %11 \n"
- "vtrn.u32 d6, d22 \n"
-
- "vtrn.u32 q4, q12 \n" // _src26ttt_r
- "vst3.u8 {d4-d6}, [%4], %11 \n"
- "vtrn.u32 d10, d26 \n"
-
- "vtrn.u32 q6, q14 \n" // _src37ttt_r
- "vst3.u8 {d8-d10}, [%3], %11 \n"
- "vtrn.u32 d14, d30 \n"
-
- "subs %0, #1 \n"
-
- "vst3.u8 {d16-d18}, [%3], %11\n"
- "vst3.u8 {d12-d14}, [%4], %11\n"
- "vst3.u8 {d20-d22}, [%4], %11\n"
- "vst3.u8 {d24-d26}, [%3], %11\n"
- "vst3.u8 {d28-d30}, [%4], %11\n"
-
- "bne 0b \n"
- : "=r"(nn), // %0
- "=r"(src0), // %1
- "=r"(src1), // %2
- "=r"(dst7), // %3
- "=r"(dst6) // %4
- : "0"(nn),
- "1"(src0),
- "2"(src1),
- "3"(dst7),
- "4"(dst6),
- "r"(src_step), // %10
- "r"(dst_step) // %11
- : "cc", "memory", "q0", "q1", "q2", "q3", "q4", "q5", "q6", "q7", "q8", "q9", "q10", "q11", "q12", "q13", "q14", "q15");
- }
- #endif // __aarch64__
- for (; remain > 0; remain--)
- {
- dst7[0] = src0[0];
- dst7[1] = src0[1];
- dst7[2] = src0[2];
- dst7[3] = src1[0];
- dst7[4] = src1[1];
- dst7[5] = src1[2];
- dst7[6] = src0[0 + src_step];
- dst7[7] = src0[1 + src_step];
- dst7[8] = src0[2 + src_step];
- dst7[9] = src1[0 + src_step];
- dst7[10] = src1[1 + src_step];
- dst7[11] = src1[2 + src_step];
- dst7[12] = src0[0 + 2 * src_step];
- dst7[13] = src0[1 + 2 * src_step];
- dst7[14] = src0[2 + 2 * src_step];
- dst7[15] = src1[0 + 2 * src_step];
- dst7[16] = src1[1 + 2 * src_step];
- dst7[17] = src1[2 + 2 * src_step];
- dst7[18] = src0[0 + 3 * src_step];
- dst7[19] = src0[1 + 3 * src_step];
- dst7[20] = src0[2 + 3 * src_step];
- dst7[21] = src1[0 + 3 * src_step];
- dst7[22] = src1[1 + 3 * src_step];
- dst7[23] = src1[2 + 3 * src_step];
-
- src0 += 3;
- src1 += 3;
-
- dst7 -= stride;
- }
-
- src0 += srcwgap + 7 * srcstride;
- }
- #endif // __ARM_NEON
- for (; y < srch; y++)
- {
- unsigned char* dst0 = dstend + y * 3;
-
- int x = 0;
- for (; x < srcw; x++)
- {
- dst0[0] = src0[0];
- dst0[1] = src0[1];
- dst0[2] = src0[2];
-
- src0 += 3;
- dst0 -= stride;
- }
-
- src0 += srcwgap;
- }
- }
-
- static void kanna_rotate_8_c4(const unsigned char* src, int srcw, int srch, int srcstride, unsigned char* dst, int /*w*/, int h, int stride)
- {
- const int srcwgap = srcstride - srcw * 4;
-
- // point to the last dst pixel row
- unsigned char* dstend = dst + stride * (h - 1);
-
- const unsigned char* src0 = src;
-
- int y = 0;
- #if __ARM_NEON
- for (; y + 7 < srch; y += 8)
- {
- const unsigned char* src1 = src0 + srcstride;
-
- unsigned char* dst7 = dstend + y * 4;
- unsigned char* dst6 = dstend + y * 4 - stride;
-
- int src_step = 2 * srcstride;
- int dst_step = -2 * stride;
-
- int nn = srcw >> 3;
- int remain = srcw - (nn << 3);
-
- #if !NCNN_GNU_INLINE_ASM || __aarch64__
- for (; nn > 0; nn--)
- {
- uint8x8x4_t _src0 = vld4_u8(src0);
- uint8x8x4_t _src1 = vld4_u8(src1);
-
- uint8x8x4_t _src2 = vld4_u8(src0 + src_step);
- uint8x8x4_t _src3 = vld4_u8(src1 + src_step);
-
- uint8x8x4_t _src4 = vld4_u8(src0 + 2 * src_step);
- uint8x8x4_t _src5 = vld4_u8(src1 + 2 * src_step);
-
- uint8x8x4_t _src6 = vld4_u8(src0 + 3 * src_step);
- uint8x8x4_t _src7 = vld4_u8(src1 + 3 * src_step);
-
- uint8x8x2_t _src01t_r = vtrn_u8(_src0.val[0], _src1.val[0]);
- uint8x8x2_t _src23t_r = vtrn_u8(_src2.val[0], _src3.val[0]);
- uint8x8x2_t _src45t_r = vtrn_u8(_src4.val[0], _src5.val[0]);
- uint8x8x2_t _src67t_r = vtrn_u8(_src6.val[0], _src7.val[0]);
-
- uint8x8x2_t _src01t_g = vtrn_u8(_src0.val[1], _src1.val[1]);
- uint8x8x2_t _src23t_g = vtrn_u8(_src2.val[1], _src3.val[1]);
- uint8x8x2_t _src45t_g = vtrn_u8(_src4.val[1], _src5.val[1]);
- uint8x8x2_t _src67t_g = vtrn_u8(_src6.val[1], _src7.val[1]);
-
- uint8x8x2_t _src01t_b = vtrn_u8(_src0.val[2], _src1.val[2]);
- uint8x8x2_t _src23t_b = vtrn_u8(_src2.val[2], _src3.val[2]);
- uint8x8x2_t _src45t_b = vtrn_u8(_src4.val[2], _src5.val[2]);
- uint8x8x2_t _src67t_b = vtrn_u8(_src6.val[2], _src7.val[2]);
-
- uint8x8x2_t _src01t_a = vtrn_u8(_src0.val[3], _src1.val[3]);
- uint8x8x2_t _src23t_a = vtrn_u8(_src2.val[3], _src3.val[3]);
- uint8x8x2_t _src45t_a = vtrn_u8(_src4.val[3], _src5.val[3]);
- uint8x8x2_t _src67t_a = vtrn_u8(_src6.val[3], _src7.val[3]);
-
- uint16x4x2_t _src02tt_r = vtrn_u16(vreinterpret_u16_u8(_src01t_r.val[0]), vreinterpret_u16_u8(_src23t_r.val[0]));
- uint16x4x2_t _src13tt_r = vtrn_u16(vreinterpret_u16_u8(_src01t_r.val[1]), vreinterpret_u16_u8(_src23t_r.val[1]));
- uint16x4x2_t _src46tt_r = vtrn_u16(vreinterpret_u16_u8(_src45t_r.val[0]), vreinterpret_u16_u8(_src67t_r.val[0]));
- uint16x4x2_t _src57tt_r = vtrn_u16(vreinterpret_u16_u8(_src45t_r.val[1]), vreinterpret_u16_u8(_src67t_r.val[1]));
-
- uint16x4x2_t _src02tt_g = vtrn_u16(vreinterpret_u16_u8(_src01t_g.val[0]), vreinterpret_u16_u8(_src23t_g.val[0]));
- uint16x4x2_t _src13tt_g = vtrn_u16(vreinterpret_u16_u8(_src01t_g.val[1]), vreinterpret_u16_u8(_src23t_g.val[1]));
- uint16x4x2_t _src46tt_g = vtrn_u16(vreinterpret_u16_u8(_src45t_g.val[0]), vreinterpret_u16_u8(_src67t_g.val[0]));
- uint16x4x2_t _src57tt_g = vtrn_u16(vreinterpret_u16_u8(_src45t_g.val[1]), vreinterpret_u16_u8(_src67t_g.val[1]));
-
- uint16x4x2_t _src02tt_b = vtrn_u16(vreinterpret_u16_u8(_src01t_b.val[0]), vreinterpret_u16_u8(_src23t_b.val[0]));
- uint16x4x2_t _src13tt_b = vtrn_u16(vreinterpret_u16_u8(_src01t_b.val[1]), vreinterpret_u16_u8(_src23t_b.val[1]));
- uint16x4x2_t _src46tt_b = vtrn_u16(vreinterpret_u16_u8(_src45t_b.val[0]), vreinterpret_u16_u8(_src67t_b.val[0]));
- uint16x4x2_t _src57tt_b = vtrn_u16(vreinterpret_u16_u8(_src45t_b.val[1]), vreinterpret_u16_u8(_src67t_b.val[1]));
-
- uint16x4x2_t _src02tt_a = vtrn_u16(vreinterpret_u16_u8(_src01t_a.val[0]), vreinterpret_u16_u8(_src23t_a.val[0]));
- uint16x4x2_t _src13tt_a = vtrn_u16(vreinterpret_u16_u8(_src01t_a.val[1]), vreinterpret_u16_u8(_src23t_a.val[1]));
- uint16x4x2_t _src46tt_a = vtrn_u16(vreinterpret_u16_u8(_src45t_a.val[0]), vreinterpret_u16_u8(_src67t_a.val[0]));
- uint16x4x2_t _src57tt_a = vtrn_u16(vreinterpret_u16_u8(_src45t_a.val[1]), vreinterpret_u16_u8(_src67t_a.val[1]));
-
- uint32x2x2_t _src04ttt_r = vtrn_u32(vreinterpret_u32_u16(_src02tt_r.val[0]), vreinterpret_u32_u16(_src46tt_r.val[0]));
- uint32x2x2_t _src15ttt_r = vtrn_u32(vreinterpret_u32_u16(_src13tt_r.val[0]), vreinterpret_u32_u16(_src57tt_r.val[0]));
- uint32x2x2_t _src26ttt_r = vtrn_u32(vreinterpret_u32_u16(_src02tt_r.val[1]), vreinterpret_u32_u16(_src46tt_r.val[1]));
- uint32x2x2_t _src37ttt_r = vtrn_u32(vreinterpret_u32_u16(_src13tt_r.val[1]), vreinterpret_u32_u16(_src57tt_r.val[1]));
-
- uint32x2x2_t _src04ttt_g = vtrn_u32(vreinterpret_u32_u16(_src02tt_g.val[0]), vreinterpret_u32_u16(_src46tt_g.val[0]));
- uint32x2x2_t _src15ttt_g = vtrn_u32(vreinterpret_u32_u16(_src13tt_g.val[0]), vreinterpret_u32_u16(_src57tt_g.val[0]));
- uint32x2x2_t _src26ttt_g = vtrn_u32(vreinterpret_u32_u16(_src02tt_g.val[1]), vreinterpret_u32_u16(_src46tt_g.val[1]));
- uint32x2x2_t _src37ttt_g = vtrn_u32(vreinterpret_u32_u16(_src13tt_g.val[1]), vreinterpret_u32_u16(_src57tt_g.val[1]));
-
- uint32x2x2_t _src04ttt_b = vtrn_u32(vreinterpret_u32_u16(_src02tt_b.val[0]), vreinterpret_u32_u16(_src46tt_b.val[0]));
- uint32x2x2_t _src15ttt_b = vtrn_u32(vreinterpret_u32_u16(_src13tt_b.val[0]), vreinterpret_u32_u16(_src57tt_b.val[0]));
- uint32x2x2_t _src26ttt_b = vtrn_u32(vreinterpret_u32_u16(_src02tt_b.val[1]), vreinterpret_u32_u16(_src46tt_b.val[1]));
- uint32x2x2_t _src37ttt_b = vtrn_u32(vreinterpret_u32_u16(_src13tt_b.val[1]), vreinterpret_u32_u16(_src57tt_b.val[1]));
-
- uint32x2x2_t _src04ttt_a = vtrn_u32(vreinterpret_u32_u16(_src02tt_a.val[0]), vreinterpret_u32_u16(_src46tt_a.val[0]));
- uint32x2x2_t _src15ttt_a = vtrn_u32(vreinterpret_u32_u16(_src13tt_a.val[0]), vreinterpret_u32_u16(_src57tt_a.val[0]));
- uint32x2x2_t _src26ttt_a = vtrn_u32(vreinterpret_u32_u16(_src02tt_a.val[1]), vreinterpret_u32_u16(_src46tt_a.val[1]));
- uint32x2x2_t _src37ttt_a = vtrn_u32(vreinterpret_u32_u16(_src13tt_a.val[1]), vreinterpret_u32_u16(_src57tt_a.val[1]));
-
- uint8x8x4_t _dst0;
- uint8x8x4_t _dst1;
- uint8x8x4_t _dst2;
- uint8x8x4_t _dst3;
- uint8x8x4_t _dst4;
- uint8x8x4_t _dst5;
- uint8x8x4_t _dst6;
- uint8x8x4_t _dst7;
-
- _dst0.val[0] = vreinterpret_u8_u32(_src04ttt_r.val[0]);
- _dst1.val[0] = vreinterpret_u8_u32(_src15ttt_r.val[0]);
- _dst2.val[0] = vreinterpret_u8_u32(_src26ttt_r.val[0]);
- _dst3.val[0] = vreinterpret_u8_u32(_src37ttt_r.val[0]);
- _dst4.val[0] = vreinterpret_u8_u32(_src04ttt_r.val[1]);
- _dst5.val[0] = vreinterpret_u8_u32(_src15ttt_r.val[1]);
- _dst6.val[0] = vreinterpret_u8_u32(_src26ttt_r.val[1]);
- _dst7.val[0] = vreinterpret_u8_u32(_src37ttt_r.val[1]);
-
- _dst0.val[1] = vreinterpret_u8_u32(_src04ttt_g.val[0]);
- _dst1.val[1] = vreinterpret_u8_u32(_src15ttt_g.val[0]);
- _dst2.val[1] = vreinterpret_u8_u32(_src26ttt_g.val[0]);
- _dst3.val[1] = vreinterpret_u8_u32(_src37ttt_g.val[0]);
- _dst4.val[1] = vreinterpret_u8_u32(_src04ttt_g.val[1]);
- _dst5.val[1] = vreinterpret_u8_u32(_src15ttt_g.val[1]);
- _dst6.val[1] = vreinterpret_u8_u32(_src26ttt_g.val[1]);
- _dst7.val[1] = vreinterpret_u8_u32(_src37ttt_g.val[1]);
-
- _dst0.val[2] = vreinterpret_u8_u32(_src04ttt_b.val[0]);
- _dst1.val[2] = vreinterpret_u8_u32(_src15ttt_b.val[0]);
- _dst2.val[2] = vreinterpret_u8_u32(_src26ttt_b.val[0]);
- _dst3.val[2] = vreinterpret_u8_u32(_src37ttt_b.val[0]);
- _dst4.val[2] = vreinterpret_u8_u32(_src04ttt_b.val[1]);
- _dst5.val[2] = vreinterpret_u8_u32(_src15ttt_b.val[1]);
- _dst6.val[2] = vreinterpret_u8_u32(_src26ttt_b.val[1]);
- _dst7.val[2] = vreinterpret_u8_u32(_src37ttt_b.val[1]);
-
- _dst0.val[3] = vreinterpret_u8_u32(_src04ttt_a.val[0]);
- _dst1.val[3] = vreinterpret_u8_u32(_src15ttt_a.val[0]);
- _dst2.val[3] = vreinterpret_u8_u32(_src26ttt_a.val[0]);
- _dst3.val[3] = vreinterpret_u8_u32(_src37ttt_a.val[0]);
- _dst4.val[3] = vreinterpret_u8_u32(_src04ttt_a.val[1]);
- _dst5.val[3] = vreinterpret_u8_u32(_src15ttt_a.val[1]);
- _dst6.val[3] = vreinterpret_u8_u32(_src26ttt_a.val[1]);
- _dst7.val[3] = vreinterpret_u8_u32(_src37ttt_a.val[1]);
-
- vst4_u8(dst7, _dst0);
- vst4_u8(dst6, _dst1);
- vst4_u8(dst7 + dst_step, _dst2);
- vst4_u8(dst6 + dst_step, _dst3);
- vst4_u8(dst7 + 2 * dst_step, _dst4);
- vst4_u8(dst6 + 2 * dst_step, _dst5);
- vst4_u8(dst7 + 3 * dst_step, _dst6);
- vst4_u8(dst6 + 3 * dst_step, _dst7);
-
- src0 += 4 * 8;
- src1 += 4 * 8;
-
- dst7 += 4 * dst_step;
- dst6 += 4 * dst_step;
- }
- #else
- if (nn > 0)
- {
- asm volatile(
- "0: \n"
- "pld [%1, #256] \n"
- "vld4.u8 {d0-d3}, [%1], %10 \n"
-
- "pld [%2, #256] \n"
- "vld4.u8 {d4-d7}, [%2], %10 \n"
-
- "pld [%1, #256] \n"
- "vld4.u8 {d8-d11}, [%1], %10 \n"
-
- "vtrn.u8 q0, q2 \n" // _src01t_r
- "vtrn.u8 q1, q3 \n"
-
- "pld [%2, #256] \n"
- "vld4.u8 {d12-d15}, [%2], %10\n"
-
- "pld [%1, #256] \n"
- "vld4.u8 {d16-d19}, [%1], %10\n"
-
- "vtrn.u8 q4, q6 \n" // _src23t_r
- "vtrn.u8 q5, q7 \n"
-
- "pld [%2, #256] \n"
- "vld4.u8 {d20-d23}, [%2], %10\n"
-
- "pld [%1, #256] \n"
- "vld4.u8 {d24-d27}, [%1], %10\n"
-
- "vtrn.u8 q8, q10 \n" // _src45t_r
- "vtrn.u8 q9, q11 \n"
-
- "pld [%2, #256] \n"
- "vld4.u8 {d28-d31}, [%2], %10\n"
-
- "vtrn.u8 q12, q14 \n" // _src67t_r
- "vtrn.u8 q13, q15 \n"
-
- "sub %1, %1, %10, lsl #2 \n" // restore src0
-
- "vtrn.u16 q0, q4 \n" // _src02tt_r
- "vtrn.u16 q1, q5 \n"
-
- "sub %2, %2, %10, lsl #2 \n" // restore src1
-
- "vtrn.u16 q2, q6 \n" // _src13tt_r
- "vtrn.u16 q3, q7 \n"
-
- "add %1, #32 \n" // src0 += 32
-
- "vtrn.u16 q8, q12 \n" // _src46tt_r
- "vtrn.u16 q9, q13 \n"
-
- "add %2, #32 \n" // src1 += 32
-
- "vtrn.u16 q10, q14 \n" // _src57tt_r
- "vtrn.u16 q11, q15 \n"
-
- "vtrn.u32 q0, q8 \n" // _src04ttt_r
- "vtrn.u32 q1, q9 \n"
-
- "vtrn.u32 q2, q10 \n" // _src15ttt_r
- "vst4.u8 {d0-d3}, [%3], %11 \n"
- "vtrn.u32 q3, q11 \n"
-
- "vtrn.u32 q4, q12 \n" // _src26ttt_r
- "vst4.u8 {d4-d7}, [%4], %11 \n"
- "vtrn.u32 q5, q13 \n"
-
- "vtrn.u32 q6, q14 \n" // _src37ttt_r
- "vst4.u8 {d8-d11}, [%3], %11 \n"
- "vtrn.u32 q7, q15 \n"
-
- "subs %0, #1 \n"
-
- "vst4.u8 {d16-d19}, [%3], %11\n"
- "vst4.u8 {d12-d15}, [%4], %11\n"
- "vst4.u8 {d20-d23}, [%4], %11\n"
- "vst4.u8 {d24-d27}, [%3], %11\n"
- "vst4.u8 {d28-d31}, [%4], %11\n"
-
- "bne 0b \n"
- : "=r"(nn), // %0
- "=r"(src0), // %1
- "=r"(src1), // %2
- "=r"(dst7), // %3
- "=r"(dst6) // %4
- : "0"(nn),
- "1"(src0),
- "2"(src1),
- "3"(dst7),
- "4"(dst6),
- "r"(src_step), // %10
- "r"(dst_step) // %11
- : "cc", "memory", "q0", "q1", "q2", "q3", "q4", "q5", "q6", "q7", "q8", "q9", "q10", "q11", "q12", "q13", "q14", "q15");
- }
- #endif // __aarch64__
- for (; remain > 0; remain--)
- {
- dst7[0] = src0[0];
- dst7[1] = src0[1];
- dst7[2] = src0[2];
- dst7[3] = src0[3];
- dst7[4] = src1[0];
- dst7[5] = src1[1];
- dst7[6] = src1[2];
- dst7[7] = src1[3];
- dst7[8] = src0[0 + src_step];
- dst7[9] = src0[1 + src_step];
- dst7[10] = src0[2 + src_step];
- dst7[11] = src0[3 + src_step];
- dst7[12] = src1[0 + src_step];
- dst7[13] = src1[1 + src_step];
- dst7[14] = src1[2 + src_step];
- dst7[15] = src1[3 + src_step];
- dst7[16] = src0[0 + 2 * src_step];
- dst7[17] = src0[1 + 2 * src_step];
- dst7[18] = src0[2 + 2 * src_step];
- dst7[19] = src0[3 + 2 * src_step];
- dst7[20] = src1[0 + 2 * src_step];
- dst7[21] = src1[1 + 2 * src_step];
- dst7[22] = src1[2 + 2 * src_step];
- dst7[23] = src1[3 + 2 * src_step];
- dst7[24] = src0[0 + 3 * src_step];
- dst7[25] = src0[1 + 3 * src_step];
- dst7[26] = src0[2 + 3 * src_step];
- dst7[27] = src0[3 + 3 * src_step];
- dst7[28] = src1[0 + 3 * src_step];
- dst7[29] = src1[1 + 3 * src_step];
- dst7[30] = src1[2 + 3 * src_step];
- dst7[31] = src1[3 + 3 * src_step];
-
- src0 += 4;
- src1 += 4;
-
- dst7 -= stride;
- }
-
- src0 += srcwgap + 7 * srcstride;
- }
- #endif // __ARM_NEON
- for (; y < srch; y++)
- {
- unsigned char* dst0 = dstend + y * 4;
-
- int x = 0;
- for (; x < srcw; x++)
- {
- dst0[0] = src0[0];
- dst0[1] = src0[1];
- dst0[2] = src0[2];
- dst0[3] = src0[3];
-
- src0 += 4;
- dst0 -= stride;
- }
-
- src0 += srcwgap;
- }
- }
-
- void kanna_rotate_c1(const unsigned char* src, int srcw, int srch, unsigned char* dst, int w, int h, int type)
- {
- return kanna_rotate_c1(src, srcw, srch, srcw, dst, w, h, w, type);
- }
-
- void kanna_rotate_c2(const unsigned char* src, int srcw, int srch, unsigned char* dst, int w, int h, int type)
- {
- return kanna_rotate_c2(src, srcw, srch, srcw * 2, dst, w, h, w * 2, type);
- }
-
- void kanna_rotate_c3(const unsigned char* src, int srcw, int srch, unsigned char* dst, int w, int h, int type)
- {
- return kanna_rotate_c3(src, srcw, srch, srcw * 3, dst, w, h, w * 3, type);
- }
-
- void kanna_rotate_c4(const unsigned char* src, int srcw, int srch, unsigned char* dst, int w, int h, int type)
- {
- return kanna_rotate_c4(src, srcw, srch, srcw * 4, dst, w, h, w * 4, type);
- }
-
- void kanna_rotate_c1(const unsigned char* src, int srcw, int srch, int srcstride, unsigned char* dst, int w, int h, int stride, int type)
- {
- // assert srcw == w && srch == h for type 1234
- // assert srcw == h && srch == w for type 5678
-
- switch (type)
- {
- case 1:
- kanna_rotate_1_c1(src, srcw, srch, srcstride, dst, w, h, stride);
- break;
- case 2:
- kanna_rotate_2_c1(src, srcw, srch, srcstride, dst, w, h, stride);
- break;
- case 3:
- kanna_rotate_3_c1(src, srcw, srch, srcstride, dst, w, h, stride);
- break;
- case 4:
- kanna_rotate_4_c1(src, srcw, srch, srcstride, dst, w, h, stride);
- break;
- case 5:
- kanna_rotate_5_c1(src, srcw, srch, srcstride, dst, w, h, stride);
- break;
- case 6:
- kanna_rotate_6_c1(src, srcw, srch, srcstride, dst, w, h, stride);
- break;
- case 7:
- kanna_rotate_7_c1(src, srcw, srch, srcstride, dst, w, h, stride);
- break;
- case 8:
- kanna_rotate_8_c1(src, srcw, srch, srcstride, dst, w, h, stride);
- break;
- default:
- // unsupported rotate type
- break;
- }
- }
-
- void kanna_rotate_c2(const unsigned char* src, int srcw, int srch, int srcstride, unsigned char* dst, int w, int h, int stride, int type)
- {
- // assert srcw == w && srch == h for type 1234
- // assert srcw == h && srch == w for type 5678
-
- switch (type)
- {
- case 1:
- kanna_rotate_1_c2(src, srcw, srch, srcstride, dst, w, h, stride);
- break;
- case 2:
- kanna_rotate_2_c2(src, srcw, srch, srcstride, dst, w, h, stride);
- break;
- case 3:
- kanna_rotate_3_c2(src, srcw, srch, srcstride, dst, w, h, stride);
- break;
- case 4:
- kanna_rotate_4_c2(src, srcw, srch, srcstride, dst, w, h, stride);
- break;
- case 5:
- kanna_rotate_5_c2(src, srcw, srch, srcstride, dst, w, h, stride);
- break;
- case 6:
- kanna_rotate_6_c2(src, srcw, srch, srcstride, dst, w, h, stride);
- break;
- case 7:
- kanna_rotate_7_c2(src, srcw, srch, srcstride, dst, w, h, stride);
- break;
- case 8:
- kanna_rotate_8_c2(src, srcw, srch, srcstride, dst, w, h, stride);
- break;
- default:
- // unsupported rotate type
- break;
- }
- }
-
- void kanna_rotate_c3(const unsigned char* src, int srcw, int srch, int srcstride, unsigned char* dst, int w, int h, int stride, int type)
- {
- // assert srcw == w && srch == h for type 1234
- // assert srcw == h && srch == w for type 5678
-
- switch (type)
- {
- case 1:
- kanna_rotate_1_c3(src, srcw, srch, srcstride, dst, w, h, stride);
- break;
- case 2:
- kanna_rotate_2_c3(src, srcw, srch, srcstride, dst, w, h, stride);
- break;
- case 3:
- kanna_rotate_3_c3(src, srcw, srch, srcstride, dst, w, h, stride);
- break;
- case 4:
- kanna_rotate_4_c3(src, srcw, srch, srcstride, dst, w, h, stride);
- break;
- case 5:
- kanna_rotate_5_c3(src, srcw, srch, srcstride, dst, w, h, stride);
- break;
- case 6:
- kanna_rotate_6_c3(src, srcw, srch, srcstride, dst, w, h, stride);
- break;
- case 7:
- kanna_rotate_7_c3(src, srcw, srch, srcstride, dst, w, h, stride);
- break;
- case 8:
- kanna_rotate_8_c3(src, srcw, srch, srcstride, dst, w, h, stride);
- break;
- default:
- // unsupported rotate type
- break;
- }
- }
-
- void kanna_rotate_c4(const unsigned char* src, int srcw, int srch, int srcstride, unsigned char* dst, int w, int h, int stride, int type)
- {
- // assert srcw == w && srch == h for type 1234
- // assert srcw == h && srch == w for type 5678
-
- switch (type)
- {
- case 1:
- kanna_rotate_1_c4(src, srcw, srch, srcstride, dst, w, h, stride);
- break;
- case 2:
- kanna_rotate_2_c4(src, srcw, srch, srcstride, dst, w, h, stride);
- break;
- case 3:
- kanna_rotate_3_c4(src, srcw, srch, srcstride, dst, w, h, stride);
- break;
- case 4:
- kanna_rotate_4_c4(src, srcw, srch, srcstride, dst, w, h, stride);
- break;
- case 5:
- kanna_rotate_5_c4(src, srcw, srch, srcstride, dst, w, h, stride);
- break;
- case 6:
- kanna_rotate_6_c4(src, srcw, srch, srcstride, dst, w, h, stride);
- break;
- case 7:
- kanna_rotate_7_c4(src, srcw, srch, srcstride, dst, w, h, stride);
- break;
- case 8:
- kanna_rotate_8_c4(src, srcw, srch, srcstride, dst, w, h, stride);
- break;
- default:
- // unsupported rotate type
- break;
- }
- }
-
- void kanna_rotate_yuv420sp(const unsigned char* src, int srcw, int srch, unsigned char* dst, int w, int h, int type)
- {
- // assert srcw % 2 == 0
- // assert srch % 2 == 0
- // assert w % 2 == 0
- // assert h % 2 == 0
-
- const unsigned char* srcY = src;
- unsigned char* dstY = dst;
- kanna_rotate_c1(srcY, srcw, srch, dstY, w, h, type);
-
- const unsigned char* srcUV = src + srcw * srch;
- unsigned char* dstUV = dst + w * h;
- kanna_rotate_c2(srcUV, srcw / 2, srch / 2, dstUV, w / 2, h / 2, type);
- }
- #endif // NCNN_PIXEL_ROTATE
-
- } // namespace ncnn
|