You can not select more than 25 topics Topics must start with a chinese character,a letter or number, can include dashes ('-') and can be up to 35 characters long.

mat_pixel_rotate.cpp 234 kB

1234567891011121314151617181920212223242526272829303132333435363738394041424344454647484950515253545556575859606162636465666768697071727374757677787980818283848586878889909192939495969798991001011021031041051061071081091101111121131141151161171181191201211221231241251261271281291301311321331341351361371381391401411421431441451461471481491501511521531541551561571581591601611621631641651661671681691701711721731741751761771781791801811821831841851861871881891901911921931941951961971981992002012022032042052062072082092102112122132142152162172182192202212222232242252262272282292302312322332342352362372382392402412422432442452462472482492502512522532542552562572582592602612622632642652662672682692702712722732742752762772782792802812822832842852862872882892902912922932942952962972982993003013023033043053063073083093103113123133143153163173183193203213223233243253263273283293303313323333343353363373383393403413423433443453463473483493503513523533543553563573583593603613623633643653663673683693703713723733743753763773783793803813823833843853863873883893903913923933943953963973983994004014024034044054064074084094104114124134144154164174184194204214224234244254264274284294304314324334344354364374384394404414424434444454464474484494504514524534544554564574584594604614624634644654664674684694704714724734744754764774784794804814824834844854864874884894904914924934944954964974984995005015025035045055065075085095105115125135145155165175185195205215225235245255265275285295305315325335345355365375385395405415425435445455465475485495505515525535545555565575585595605615625635645655665675685695705715725735745755765775785795805815825835845855865875885895905915925935945955965975985996006016026036046056066076086096106116126136146156166176186196206216226236246256266276286296306316326336346356366376386396406416426436446456466476486496506516526536546556566576586596606616626636646656666676686696706716726736746756766776786796806816826836846856866876886896906916926936946956966976986997007017027037047057067077087097107117127137147157167177187197207217227237247257267277287297307317327337347357367377387397407417427437447457467477487497507517527537547557567577587597607617627637647657667677687697707717727737747757767777787797807817827837847857867877887897907917927937947957967977987998008018028038048058068078088098108118128138148158168178188198208218228238248258268278288298308318328338348358368378388398408418428438448458468478488498508518528538548558568578588598608618628638648658668678688698708718728738748758768778788798808818828838848858868878888898908918928938948958968978988999009019029039049059069079089099109119129139149159169179189199209219229239249259269279289299309319329339349359369379389399409419429439449459469479489499509519529539549559569579589599609619629639649659669679689699709719729739749759769779789799809819829839849859869879889899909919929939949959969979989991000100110021003100410051006100710081009101010111012101310141015101610171018101910201021102210231024102510261027102810291030103110321033103410351036103710381039104010411042104310441045104610471048104910501051105210531054105510561057105810591060106110621063106410651066106710681069107010711072107310741075107610771078107910801081108210831084108510861087108810891090109110921093109410951096109710981099110011011102110311041105110611071108110911101111111211131114111511161117111811191120112111221123112411251126112711281129113011311132113311341135113611371138113911401141114211431144114511461147114811491150115111521153115411551156115711581159116011611162116311641165116611671168116911701171117211731174117511761177117811791180118111821183118411851186118711881189119011911192119311941195119611971198119912001201120212031204120512061207120812091210121112121213121412151216121712181219122012211222122312241225122612271228122912301231123212331234123512361237123812391240124112421243124412451246124712481249125012511252125312541255125612571258125912601261126212631264126512661267126812691270127112721273127412751276127712781279128012811282128312841285128612871288128912901291129212931294129512961297129812991300130113021303130413051306130713081309131013111312131313141315131613171318131913201321132213231324132513261327132813291330133113321333133413351336133713381339134013411342134313441345134613471348134913501351135213531354135513561357135813591360136113621363136413651366136713681369137013711372137313741375137613771378137913801381138213831384138513861387138813891390139113921393139413951396139713981399140014011402140314041405140614071408140914101411141214131414141514161417141814191420142114221423142414251426142714281429143014311432143314341435143614371438143914401441144214431444144514461447144814491450145114521453145414551456145714581459146014611462146314641465146614671468146914701471147214731474147514761477147814791480148114821483148414851486148714881489149014911492149314941495149614971498149915001501150215031504150515061507150815091510151115121513151415151516151715181519152015211522152315241525152615271528152915301531153215331534153515361537153815391540154115421543154415451546154715481549155015511552155315541555155615571558155915601561156215631564156515661567156815691570157115721573157415751576157715781579158015811582158315841585158615871588158915901591159215931594159515961597159815991600160116021603160416051606160716081609161016111612161316141615161616171618161916201621162216231624162516261627162816291630163116321633163416351636163716381639164016411642164316441645164616471648164916501651165216531654165516561657165816591660166116621663166416651666166716681669167016711672167316741675167616771678167916801681168216831684168516861687168816891690169116921693169416951696169716981699170017011702170317041705170617071708170917101711171217131714171517161717171817191720172117221723172417251726172717281729173017311732173317341735173617371738173917401741174217431744174517461747174817491750175117521753175417551756175717581759176017611762176317641765176617671768176917701771177217731774177517761777177817791780178117821783178417851786178717881789179017911792179317941795179617971798179918001801180218031804180518061807180818091810181118121813181418151816181718181819182018211822182318241825182618271828182918301831183218331834183518361837183818391840184118421843184418451846184718481849185018511852185318541855185618571858185918601861186218631864186518661867186818691870187118721873187418751876187718781879188018811882188318841885188618871888188918901891189218931894189518961897189818991900190119021903190419051906190719081909191019111912191319141915191619171918191919201921192219231924192519261927192819291930193119321933193419351936193719381939194019411942194319441945194619471948194919501951195219531954195519561957195819591960196119621963196419651966196719681969197019711972197319741975197619771978197919801981198219831984198519861987198819891990199119921993199419951996199719981999200020012002200320042005200620072008200920102011201220132014201520162017201820192020202120222023202420252026202720282029203020312032203320342035203620372038203920402041204220432044204520462047204820492050205120522053205420552056205720582059206020612062206320642065206620672068206920702071207220732074207520762077207820792080208120822083208420852086208720882089209020912092209320942095209620972098209921002101210221032104210521062107210821092110211121122113211421152116211721182119212021212122212321242125212621272128212921302131213221332134213521362137213821392140214121422143214421452146214721482149215021512152215321542155215621572158215921602161216221632164216521662167216821692170217121722173217421752176217721782179218021812182218321842185218621872188218921902191219221932194219521962197219821992200220122022203220422052206220722082209221022112212221322142215221622172218221922202221222222232224222522262227222822292230223122322233223422352236223722382239224022412242224322442245224622472248224922502251225222532254225522562257225822592260226122622263226422652266226722682269227022712272227322742275227622772278227922802281228222832284228522862287228822892290229122922293229422952296229722982299230023012302230323042305230623072308230923102311231223132314231523162317231823192320232123222323232423252326232723282329233023312332233323342335233623372338233923402341234223432344234523462347234823492350235123522353235423552356235723582359236023612362236323642365236623672368236923702371237223732374237523762377237823792380238123822383238423852386238723882389239023912392239323942395239623972398239924002401240224032404240524062407240824092410241124122413241424152416241724182419242024212422242324242425242624272428242924302431243224332434243524362437243824392440244124422443244424452446244724482449245024512452245324542455245624572458245924602461246224632464246524662467246824692470247124722473247424752476247724782479248024812482248324842485248624872488248924902491249224932494249524962497249824992500250125022503250425052506250725082509251025112512251325142515251625172518251925202521252225232524252525262527252825292530253125322533253425352536253725382539254025412542254325442545254625472548254925502551255225532554255525562557255825592560256125622563256425652566256725682569257025712572257325742575257625772578257925802581258225832584258525862587258825892590259125922593259425952596259725982599260026012602260326042605260626072608260926102611261226132614261526162617261826192620262126222623262426252626262726282629263026312632263326342635263626372638263926402641264226432644264526462647264826492650265126522653265426552656265726582659266026612662266326642665266626672668266926702671267226732674267526762677267826792680268126822683268426852686268726882689269026912692269326942695269626972698269927002701270227032704270527062707270827092710271127122713271427152716271727182719272027212722272327242725272627272728272927302731273227332734273527362737273827392740274127422743274427452746274727482749275027512752275327542755275627572758275927602761276227632764276527662767276827692770277127722773277427752776277727782779278027812782278327842785278627872788278927902791279227932794279527962797279827992800280128022803280428052806280728082809281028112812281328142815281628172818281928202821282228232824282528262827282828292830283128322833283428352836283728382839284028412842284328442845284628472848284928502851285228532854285528562857285828592860286128622863286428652866286728682869287028712872287328742875287628772878287928802881288228832884288528862887288828892890289128922893289428952896289728982899290029012902290329042905290629072908290929102911291229132914291529162917291829192920292129222923292429252926292729282929293029312932293329342935293629372938293929402941294229432944294529462947294829492950295129522953295429552956295729582959296029612962296329642965296629672968296929702971297229732974297529762977297829792980298129822983298429852986298729882989299029912992299329942995299629972998299930003001300230033004300530063007300830093010301130123013301430153016301730183019302030213022302330243025302630273028302930303031303230333034303530363037303830393040304130423043304430453046304730483049305030513052305330543055305630573058305930603061306230633064306530663067306830693070307130723073307430753076307730783079308030813082308330843085308630873088308930903091309230933094309530963097309830993100310131023103310431053106310731083109311031113112311331143115311631173118311931203121312231233124312531263127312831293130313131323133313431353136313731383139314031413142314331443145314631473148314931503151315231533154315531563157315831593160316131623163316431653166316731683169317031713172317331743175317631773178317931803181318231833184318531863187318831893190319131923193319431953196319731983199320032013202320332043205320632073208320932103211321232133214321532163217321832193220322132223223322432253226322732283229323032313232323332343235323632373238323932403241324232433244324532463247324832493250325132523253325432553256325732583259326032613262326332643265326632673268326932703271327232733274327532763277327832793280328132823283328432853286328732883289329032913292329332943295329632973298329933003301330233033304330533063307330833093310331133123313331433153316331733183319332033213322332333243325332633273328332933303331333233333334333533363337333833393340334133423343334433453346334733483349335033513352335333543355335633573358335933603361336233633364336533663367336833693370337133723373337433753376337733783379338033813382338333843385338633873388338933903391339233933394339533963397339833993400340134023403340434053406340734083409341034113412341334143415341634173418341934203421342234233424342534263427342834293430343134323433343434353436343734383439344034413442344334443445344634473448344934503451345234533454345534563457345834593460346134623463346434653466346734683469347034713472347334743475347634773478347934803481348234833484348534863487348834893490349134923493349434953496349734983499350035013502350335043505350635073508350935103511351235133514351535163517351835193520352135223523352435253526352735283529353035313532353335343535353635373538353935403541354235433544354535463547354835493550355135523553355435553556355735583559356035613562356335643565356635673568356935703571357235733574357535763577357835793580358135823583358435853586358735883589359035913592359335943595359635973598359936003601360236033604360536063607360836093610361136123613361436153616361736183619362036213622362336243625362636273628362936303631363236333634363536363637363836393640364136423643364436453646364736483649365036513652365336543655365636573658365936603661366236633664366536663667366836693670367136723673367436753676367736783679368036813682368336843685368636873688368936903691369236933694369536963697369836993700370137023703370437053706370737083709371037113712371337143715371637173718371937203721372237233724372537263727372837293730373137323733373437353736373737383739374037413742374337443745374637473748374937503751375237533754375537563757375837593760376137623763376437653766376737683769377037713772377337743775377637773778377937803781378237833784378537863787378837893790379137923793379437953796379737983799380038013802380338043805380638073808380938103811381238133814381538163817381838193820382138223823382438253826382738283829383038313832383338343835383638373838383938403841384238433844384538463847384838493850385138523853385438553856385738583859386038613862386338643865386638673868386938703871387238733874387538763877387838793880388138823883388438853886388738883889389038913892389338943895389638973898389939003901390239033904390539063907390839093910391139123913391439153916391739183919392039213922392339243925392639273928392939303931393239333934393539363937393839393940394139423943394439453946394739483949395039513952395339543955395639573958395939603961396239633964396539663967396839693970397139723973397439753976397739783979398039813982398339843985398639873988398939903991399239933994399539963997399839994000400140024003400440054006400740084009401040114012401340144015401640174018401940204021402240234024402540264027402840294030403140324033403440354036403740384039404040414042404340444045404640474048404940504051405240534054405540564057405840594060406140624063406440654066406740684069407040714072407340744075407640774078407940804081408240834084408540864087408840894090409140924093409440954096409740984099410041014102410341044105410641074108410941104111411241134114411541164117411841194120412141224123412441254126412741284129413041314132413341344135413641374138413941404141414241434144414541464147414841494150415141524153415441554156415741584159416041614162416341644165416641674168416941704171417241734174417541764177417841794180418141824183418441854186418741884189419041914192419341944195419641974198419942004201420242034204420542064207420842094210421142124213421442154216421742184219422042214222422342244225422642274228422942304231423242334234423542364237423842394240424142424243424442454246424742484249425042514252425342544255425642574258425942604261426242634264426542664267426842694270427142724273427442754276427742784279428042814282428342844285428642874288428942904291429242934294429542964297429842994300430143024303430443054306430743084309431043114312431343144315431643174318431943204321432243234324432543264327432843294330433143324333433443354336433743384339434043414342434343444345434643474348434943504351435243534354435543564357435843594360436143624363436443654366436743684369437043714372437343744375437643774378437943804381438243834384438543864387438843894390439143924393439443954396439743984399440044014402440344044405440644074408440944104411441244134414441544164417441844194420442144224423442444254426442744284429443044314432443344344435443644374438443944404441444244434444444544464447444844494450445144524453445444554456445744584459446044614462446344644465446644674468446944704471447244734474447544764477447844794480448144824483448444854486448744884489449044914492449344944495449644974498449945004501450245034504450545064507450845094510451145124513451445154516451745184519452045214522452345244525452645274528452945304531453245334534453545364537453845394540454145424543454445454546454745484549455045514552455345544555455645574558455945604561456245634564456545664567456845694570457145724573457445754576457745784579458045814582458345844585458645874588458945904591459245934594459545964597459845994600460146024603460446054606460746084609461046114612461346144615461646174618461946204621462246234624462546264627462846294630463146324633463446354636463746384639464046414642464346444645464646474648464946504651465246534654465546564657465846594660466146624663466446654666466746684669467046714672467346744675467646774678467946804681468246834684468546864687468846894690469146924693469446954696469746984699470047014702470347044705470647074708470947104711471247134714471547164717471847194720472147224723472447254726472747284729473047314732473347344735473647374738473947404741474247434744474547464747474847494750475147524753475447554756475747584759476047614762476347644765476647674768476947704771477247734774477547764777477847794780478147824783478447854786478747884789479047914792479347944795479647974798479948004801480248034804480548064807480848094810481148124813481448154816481748184819482048214822482348244825482648274828482948304831483248334834483548364837483848394840484148424843484448454846484748484849485048514852485348544855485648574858485948604861486248634864486548664867486848694870487148724873487448754876487748784879488048814882488348844885488648874888488948904891489248934894489548964897489848994900490149024903490449054906490749084909491049114912491349144915491649174918491949204921492249234924492549264927492849294930493149324933493449354936493749384939494049414942494349444945494649474948494949504951495249534954495549564957495849594960496149624963496449654966496749684969497049714972497349744975497649774978497949804981498249834984498549864987498849894990499149924993499449954996499749984999500050015002500350045005500650075008500950105011501250135014501550165017501850195020502150225023502450255026502750285029503050315032503350345035503650375038503950405041504250435044504550465047504850495050505150525053505450555056505750585059506050615062506350645065506650675068506950705071507250735074507550765077507850795080508150825083508450855086508750885089509050915092509350945095509650975098509951005101510251035104510551065107510851095110511151125113511451155116511751185119512051215122512351245125512651275128512951305131513251335134513551365137513851395140514151425143514451455146514751485149515051515152515351545155515651575158515951605161516251635164516551665167516851695170517151725173517451755176517751785179518051815182518351845185518651875188518951905191519251935194519551965197519851995200520152025203520452055206520752085209521052115212521352145215521652175218521952205221522252235224522552265227522852295230523152325233523452355236523752385239524052415242524352445245524652475248524952505251525252535254525552565257525852595260526152625263526452655266526752685269527052715272527352745275527652775278527952805281528252835284528552865287528852895290529152925293529452955296529752985299530053015302530353045305530653075308530953105311531253135314531553165317531853195320532153225323532453255326532753285329533053315332533353345335533653375338533953405341534253435344534553465347534853495350535153525353535453555356535753585359536053615362536353645365536653675368536953705371537253735374537553765377537853795380538153825383538453855386538753885389539053915392539353945395539653975398539954005401540254035404540554065407540854095410541154125413541454155416541754185419542054215422542354245425542654275428542954305431543254335434543554365437543854395440544154425443544454455446544754485449545054515452545354545455545654575458545954605461546254635464546554665467546854695470547154725473547454755476547754785479548054815482548354845485548654875488548954905491549254935494549554965497549854995500550155025503550455055506550755085509551055115512551355145515551655175518551955205521552255235524552555265527552855295530553155325533553455355536553755385539554055415542554355445545554655475548554955505551555255535554555555565557555855595560556155625563556455655566556755685569557055715572557355745575557655775578557955805581558255835584558555865587558855895590559155925593559455955596559755985599560056015602560356045605560656075608560956105611561256135614561556165617561856195620562156225623562456255626562756285629563056315632563356345635563656375638563956405641564256435644564556465647564856495650565156525653565456555656565756585659566056615662566356645665566656675668566956705671567256735674567556765677567856795680568156825683568456855686568756885689569056915692569356945695569656975698569957005701570257035704570557065707570857095710571157125713571457155716571757185719572057215722572357245725572657275728572957305731573257335734573557365737573857395740574157425743574457455746574757485749575057515752575357545755575657575758575957605761576257635764576557665767576857695770577157725773577457755776577757785779578057815782578357845785578657875788578957905791579257935794579557965797579857995800580158025803580458055806580758085809581058115812581358145815581658175818581958205821582258235824582558265827582858295830583158325833583458355836583758385839584058415842584358445845584658475848584958505851585258535854585558565857585858595860586158625863586458655866586758685869587058715872587358745875587658775878587958805881588258835884588558865887588858895890589158925893589458955896589758985899590059015902590359045905590659075908590959105911591259135914591559165917591859195920592159225923592459255926592759285929593059315932593359345935593659375938593959405941594259435944594559465947594859495950595159525953595459555956595759585959596059615962596359645965596659675968596959705971597259735974597559765977597859795980598159825983598459855986598759885989599059915992599359945995599659975998599960006001600260036004600560066007600860096010601160126013601460156016601760186019602060216022602360246025602660276028602960306031603260336034603560366037603860396040604160426043604460456046604760486049605060516052605360546055605660576058605960606061606260636064606560666067606860696070607160726073607460756076607760786079608060816082608360846085608660876088608960906091609260936094609560966097609860996100610161026103610461056106
  1. // Copyright 2019 Tencent
  2. // SPDX-License-Identifier: BSD-3-Clause
  3. #include "mat.h"
  4. #if __ARM_NEON
  5. #include <arm_neon.h>
  6. #endif // __ARM_NEON
  7. #include "platform.h"
  8. namespace ncnn {
  9. #if NCNN_PIXEL_ROTATE
  10. // should be a kanna ascii art here in my local branch
  11. // but we shall ask the original art author for permission first ...
  12. // https://www.reddit.com/r/anime/comments/5uxjn4/i_recreated_the_kanna_ascii_art_from_kobayashisan/
  13. static void kanna_rotate_1_c1(const unsigned char* src, int srcw, int srch, int srcstride, unsigned char* dst, int w, int /*h*/, int stride)
  14. {
  15. const int srcwgap = srcstride - srcw;
  16. const int wgap = stride - w;
  17. const unsigned char* src0 = src;
  18. const unsigned char* src1 = src + srcstride;
  19. unsigned char* dst0 = dst;
  20. unsigned char* dst1 = dst + stride;
  21. int y = 0;
  22. for (; y + 1 < srch; y += 2)
  23. {
  24. #if __ARM_NEON
  25. int nn = srcw >> 5;
  26. int remain = srcw - (nn << 5);
  27. #if !NCNN_GNU_INLINE_ASM || __aarch64__
  28. for (; nn > 0; nn--)
  29. {
  30. uint8x16_t _src0 = vld1q_u8(src0);
  31. uint8x16_t _src0n = vld1q_u8(src0 + 16);
  32. vst1q_u8(dst0, _src0);
  33. vst1q_u8(dst0 + 16, _src0n);
  34. uint8x16_t _src1 = vld1q_u8(src1);
  35. uint8x16_t _src1n = vld1q_u8(src1 + 16);
  36. vst1q_u8(dst1, _src1);
  37. vst1q_u8(dst1 + 16, _src1n);
  38. src0 += 32;
  39. src1 += 32;
  40. dst0 += 32;
  41. dst1 += 32;
  42. }
  43. #else
  44. if (nn > 0)
  45. {
  46. asm volatile(
  47. "0: \n"
  48. "pld [%1, #256] \n"
  49. "vld1.u8 {d0-d3}, [%1]! \n"
  50. "pld [%2, #256] \n"
  51. "vld1.u8 {d4-d7}, [%2]! \n"
  52. "subs %0, #1 \n"
  53. "vst1.u8 {d0-d3}, [%3]! \n"
  54. "vst1.u8 {d4-d7}, [%4]! \n"
  55. "bne 0b \n"
  56. : "=r"(nn), // %0
  57. "=r"(src0), // %1
  58. "=r"(src1), // %2
  59. "=r"(dst0), // %3
  60. "=r"(dst1) // %4
  61. : "0"(nn),
  62. "1"(src0),
  63. "2"(src1),
  64. "3"(dst0),
  65. "4"(dst1)
  66. : "cc", "memory", "q0", "q1", "q2", "q3");
  67. }
  68. #endif // __aarch64__
  69. #else
  70. int remain = srcw;
  71. #endif // __ARM_NEON
  72. for (; remain > 0; remain--)
  73. {
  74. *dst0++ = *src0++;
  75. *dst1++ = *src1++;
  76. }
  77. src0 += srcwgap + srcstride;
  78. src1 += srcwgap + srcstride;
  79. dst0 += wgap + stride;
  80. dst1 += wgap + stride;
  81. }
  82. for (; y < srch; y++)
  83. {
  84. #if __ARM_NEON
  85. int nn = srcw >> 5;
  86. int remain = srcw - (nn << 5);
  87. #if !NCNN_GNU_INLINE_ASM || __aarch64__
  88. for (; nn > 0; nn--)
  89. {
  90. uint8x16_t _src = vld1q_u8(src0);
  91. uint8x16_t _src2 = vld1q_u8(src0 + 16);
  92. vst1q_u8(dst0, _src);
  93. vst1q_u8(dst0 + 16, _src2);
  94. src0 += 32;
  95. dst0 += 32;
  96. }
  97. #else
  98. if (nn > 0)
  99. {
  100. asm volatile(
  101. "0: \n"
  102. "pld [%1, #256] \n"
  103. "vld1.u8 {d0-d3}, [%1]! \n"
  104. "subs %0, #1 \n"
  105. "vst1.u8 {d0-d3}, [%2]! \n"
  106. "bne 0b \n"
  107. : "=r"(nn), // %0
  108. "=r"(src0), // %1
  109. "=r"(dst0) // %2
  110. : "0"(nn),
  111. "1"(src0),
  112. "2"(dst0)
  113. : "cc", "memory", "q0", "q1");
  114. }
  115. #endif // __aarch64__
  116. #else
  117. int remain = srcw;
  118. #endif // __ARM_NEON
  119. for (; remain > 0; remain--)
  120. {
  121. *dst0++ = *src0++;
  122. }
  123. src0 += srcwgap;
  124. dst0 += wgap;
  125. }
  126. }
  127. static void kanna_rotate_1_c2(const unsigned char* src, int srcw, int srch, int srcstride, unsigned char* dst, int w, int /*h*/, int stride)
  128. {
  129. const int srcwgap = srcstride - srcw * 2;
  130. const int wgap = stride - w * 2;
  131. int size = srcw * 2;
  132. const unsigned char* src0 = src;
  133. const unsigned char* src1 = src + srcstride;
  134. unsigned char* dst0 = dst;
  135. unsigned char* dst1 = dst + stride;
  136. int y = 0;
  137. for (; y + 1 < srch; y += 2)
  138. {
  139. #if __ARM_NEON
  140. int nn = size >> 5;
  141. int remain = size - (nn << 5);
  142. #if !NCNN_GNU_INLINE_ASM || __aarch64__
  143. for (; nn > 0; nn--)
  144. {
  145. uint8x16_t _src0 = vld1q_u8(src0);
  146. uint8x16_t _src0n = vld1q_u8(src0 + 16);
  147. vst1q_u8(dst0, _src0);
  148. vst1q_u8(dst0 + 16, _src0n);
  149. uint8x16_t _src1 = vld1q_u8(src1);
  150. uint8x16_t _src1n = vld1q_u8(src1 + 16);
  151. vst1q_u8(dst1, _src1);
  152. vst1q_u8(dst1 + 16, _src1n);
  153. src0 += 32;
  154. src1 += 32;
  155. dst0 += 32;
  156. dst1 += 32;
  157. }
  158. #else
  159. if (nn > 0)
  160. {
  161. asm volatile(
  162. "0: \n"
  163. "pld [%1, #256] \n"
  164. "vld1.u8 {d0-d3}, [%1]! \n"
  165. "pld [%2, #256] \n"
  166. "vld1.u8 {d4-d7}, [%2]! \n"
  167. "subs %0, #1 \n"
  168. "vst1.u8 {d0-d3}, [%3]! \n"
  169. "vst1.u8 {d4-d7}, [%4]! \n"
  170. "bne 0b \n"
  171. : "=r"(nn), // %0
  172. "=r"(src0), // %1
  173. "=r"(src1), // %2
  174. "=r"(dst0), // %3
  175. "=r"(dst1) // %4
  176. : "0"(nn),
  177. "1"(src0),
  178. "2"(src1),
  179. "3"(dst0),
  180. "4"(dst1)
  181. : "cc", "memory", "q0", "q1", "q2", "q3");
  182. }
  183. #endif // __aarch64__
  184. #else
  185. int remain = size;
  186. #endif // __ARM_NEON
  187. for (; remain > 0; remain--)
  188. {
  189. *dst0++ = *src0++;
  190. *dst1++ = *src1++;
  191. }
  192. src0 += srcwgap + srcstride;
  193. src1 += srcwgap + srcstride;
  194. dst0 += wgap + stride;
  195. dst1 += wgap + stride;
  196. }
  197. for (; y < srch; y++)
  198. {
  199. #if __ARM_NEON
  200. int nn = size >> 5;
  201. int remain = size - (nn << 5);
  202. #if !NCNN_GNU_INLINE_ASM || __aarch64__
  203. for (; nn > 0; nn--)
  204. {
  205. uint8x16_t _src = vld1q_u8(src0);
  206. uint8x16_t _src2 = vld1q_u8(src0 + 16);
  207. vst1q_u8(dst0, _src);
  208. vst1q_u8(dst0 + 16, _src2);
  209. src0 += 32;
  210. dst0 += 32;
  211. }
  212. #else
  213. if (nn > 0)
  214. {
  215. asm volatile(
  216. "0: \n"
  217. "pld [%1, #256] \n"
  218. "vld1.u8 {d0-d3}, [%1]! \n"
  219. "subs %0, #1 \n"
  220. "vst1.u8 {d0-d3}, [%2]! \n"
  221. "bne 0b \n"
  222. : "=r"(nn), // %0
  223. "=r"(src0), // %1
  224. "=r"(dst0) // %2
  225. : "0"(nn),
  226. "1"(src0),
  227. "2"(dst0)
  228. : "cc", "memory", "q0", "q1");
  229. }
  230. #endif // __aarch64__
  231. #else
  232. int remain = size;
  233. #endif // __ARM_NEON
  234. for (; remain > 0; remain--)
  235. {
  236. *dst0++ = *src0++;
  237. }
  238. src0 += srcwgap;
  239. dst0 += wgap;
  240. }
  241. }
  242. static void kanna_rotate_1_c3(const unsigned char* src, int srcw, int srch, int srcstride, unsigned char* dst, int w, int /*h*/, int stride)
  243. {
  244. const int srcwgap = srcstride - srcw * 3;
  245. const int wgap = stride - w * 3;
  246. int size = srcw * 3;
  247. const unsigned char* src0 = src;
  248. const unsigned char* src1 = src + srcstride;
  249. unsigned char* dst0 = dst;
  250. unsigned char* dst1 = dst + stride;
  251. int y = 0;
  252. for (; y + 1 < srch; y += 2)
  253. {
  254. #if __ARM_NEON
  255. int nn = size >> 5;
  256. int remain = size - (nn << 5);
  257. #if !NCNN_GNU_INLINE_ASM || __aarch64__
  258. for (; nn > 0; nn--)
  259. {
  260. uint8x16_t _src0 = vld1q_u8(src0);
  261. uint8x16_t _src0n = vld1q_u8(src0 + 16);
  262. vst1q_u8(dst0, _src0);
  263. vst1q_u8(dst0 + 16, _src0n);
  264. uint8x16_t _src1 = vld1q_u8(src1);
  265. uint8x16_t _src1n = vld1q_u8(src1 + 16);
  266. vst1q_u8(dst1, _src1);
  267. vst1q_u8(dst1 + 16, _src1n);
  268. src0 += 32;
  269. src1 += 32;
  270. dst0 += 32;
  271. dst1 += 32;
  272. }
  273. #else
  274. if (nn > 0)
  275. {
  276. asm volatile(
  277. "0: \n"
  278. "pld [%1, #256] \n"
  279. "vld1.u8 {d0-d3}, [%1]! \n"
  280. "pld [%2, #256] \n"
  281. "vld1.u8 {d4-d7}, [%2]! \n"
  282. "subs %0, #1 \n"
  283. "vst1.u8 {d0-d3}, [%3]! \n"
  284. "vst1.u8 {d4-d7}, [%4]! \n"
  285. "bne 0b \n"
  286. : "=r"(nn), // %0
  287. "=r"(src0), // %1
  288. "=r"(src1), // %2
  289. "=r"(dst0), // %3
  290. "=r"(dst1) // %4
  291. : "0"(nn),
  292. "1"(src0),
  293. "2"(src1),
  294. "3"(dst0),
  295. "4"(dst1)
  296. : "cc", "memory", "q0", "q1", "q2", "q3");
  297. }
  298. #endif // __aarch64__
  299. #else
  300. int remain = size;
  301. #endif // __ARM_NEON
  302. for (; remain > 0; remain--)
  303. {
  304. *dst0++ = *src0++;
  305. *dst1++ = *src1++;
  306. }
  307. src0 += srcwgap + srcstride;
  308. src1 += srcwgap + srcstride;
  309. dst0 += wgap + stride;
  310. dst1 += wgap + stride;
  311. }
  312. for (; y < srch; y++)
  313. {
  314. #if __ARM_NEON
  315. int nn = size >> 5;
  316. int remain = size - (nn << 5);
  317. #if !NCNN_GNU_INLINE_ASM || __aarch64__
  318. for (; nn > 0; nn--)
  319. {
  320. uint8x16_t _src = vld1q_u8(src0);
  321. uint8x16_t _src2 = vld1q_u8(src0 + 16);
  322. vst1q_u8(dst0, _src);
  323. vst1q_u8(dst0 + 16, _src2);
  324. src0 += 32;
  325. dst0 += 32;
  326. }
  327. #else
  328. if (nn > 0)
  329. {
  330. asm volatile(
  331. "0: \n"
  332. "pld [%1, #256] \n"
  333. "vld1.u8 {d0-d3}, [%1]! \n"
  334. "subs %0, #1 \n"
  335. "vst1.u8 {d0-d3}, [%2]! \n"
  336. "bne 0b \n"
  337. : "=r"(nn), // %0
  338. "=r"(src0), // %1
  339. "=r"(dst0) // %2
  340. : "0"(nn),
  341. "1"(src0),
  342. "2"(dst0)
  343. : "cc", "memory", "q0", "q1");
  344. }
  345. #endif // __aarch64__
  346. #else
  347. int remain = size;
  348. #endif // __ARM_NEON
  349. for (; remain > 0; remain--)
  350. {
  351. *dst0++ = *src0++;
  352. }
  353. src0 += srcwgap;
  354. dst0 += wgap;
  355. }
  356. }
  357. static void kanna_rotate_1_c4(const unsigned char* src, int srcw, int srch, int srcstride, unsigned char* dst, int w, int /*h*/, int stride)
  358. {
  359. const int srcwgap = srcstride - srcw * 4;
  360. const int wgap = stride - w * 4;
  361. int size = srcw * 4;
  362. const unsigned char* src0 = src;
  363. const unsigned char* src1 = src + srcstride;
  364. unsigned char* dst0 = dst;
  365. unsigned char* dst1 = dst + stride;
  366. int y = 0;
  367. for (; y + 1 < srch; y += 2)
  368. {
  369. #if __ARM_NEON
  370. int nn = size >> 5;
  371. int remain = size - (nn << 5);
  372. #if !NCNN_GNU_INLINE_ASM || __aarch64__
  373. for (; nn > 0; nn--)
  374. {
  375. uint8x16_t _src0 = vld1q_u8(src0);
  376. uint8x16_t _src0n = vld1q_u8(src0 + 16);
  377. vst1q_u8(dst0, _src0);
  378. vst1q_u8(dst0 + 16, _src0n);
  379. uint8x16_t _src1 = vld1q_u8(src1);
  380. uint8x16_t _src1n = vld1q_u8(src1 + 16);
  381. vst1q_u8(dst1, _src1);
  382. vst1q_u8(dst1 + 16, _src1n);
  383. src0 += 32;
  384. src1 += 32;
  385. dst0 += 32;
  386. dst1 += 32;
  387. }
  388. #else
  389. if (nn > 0)
  390. {
  391. asm volatile(
  392. "0: \n"
  393. "pld [%1, #256] \n"
  394. "vld1.u8 {d0-d3}, [%1]! \n"
  395. "pld [%2, #256] \n"
  396. "vld1.u8 {d4-d7}, [%2]! \n"
  397. "subs %0, #1 \n"
  398. "vst1.u8 {d0-d3}, [%3]! \n"
  399. "vst1.u8 {d4-d7}, [%4]! \n"
  400. "bne 0b \n"
  401. : "=r"(nn), // %0
  402. "=r"(src0), // %1
  403. "=r"(src1), // %2
  404. "=r"(dst0), // %3
  405. "=r"(dst1) // %4
  406. : "0"(nn),
  407. "1"(src0),
  408. "2"(src1),
  409. "3"(dst0),
  410. "4"(dst1)
  411. : "cc", "memory", "q0", "q1", "q2", "q3");
  412. }
  413. #endif // __aarch64__
  414. #else
  415. int remain = size;
  416. #endif // __ARM_NEON
  417. for (; remain > 0; remain--)
  418. {
  419. *dst0++ = *src0++;
  420. *dst1++ = *src1++;
  421. }
  422. src0 += srcwgap + srcstride;
  423. src1 += srcwgap + srcstride;
  424. dst0 += wgap + stride;
  425. dst1 += wgap + stride;
  426. }
  427. for (; y < srch; y++)
  428. {
  429. #if __ARM_NEON
  430. int nn = size >> 5;
  431. int remain = size - (nn << 5);
  432. #if !NCNN_GNU_INLINE_ASM || __aarch64__
  433. for (; nn > 0; nn--)
  434. {
  435. uint8x16_t _src = vld1q_u8(src0);
  436. uint8x16_t _src2 = vld1q_u8(src0 + 16);
  437. vst1q_u8(dst0, _src);
  438. vst1q_u8(dst0 + 16, _src2);
  439. src0 += 32;
  440. dst0 += 32;
  441. }
  442. #else
  443. if (nn > 0)
  444. {
  445. asm volatile(
  446. "0: \n"
  447. "pld [%1, #256] \n"
  448. "vld1.u8 {d0-d3}, [%1]! \n"
  449. "subs %0, #1 \n"
  450. "vst1.u8 {d0-d3}, [%2]! \n"
  451. "bne 0b \n"
  452. : "=r"(nn), // %0
  453. "=r"(src0), // %1
  454. "=r"(dst0) // %2
  455. : "0"(nn),
  456. "1"(src0),
  457. "2"(dst0)
  458. : "cc", "memory", "q0", "q1");
  459. }
  460. #endif // __aarch64__
  461. #else
  462. int remain = size;
  463. #endif // __ARM_NEON
  464. for (; remain > 0; remain--)
  465. {
  466. *dst0++ = *src0++;
  467. }
  468. src0 += srcwgap;
  469. dst0 += wgap;
  470. }
  471. }
  472. static void kanna_rotate_2_c1(const unsigned char* src, int srcw, int srch, int srcstride, unsigned char* dst, int w, int /*h*/, int stride)
  473. {
  474. const int srcwgap = srcstride - srcw;
  475. const int wgap = stride + w;
  476. const unsigned char* src0 = src;
  477. unsigned char* dst0 = dst + w - 1;
  478. int y = 0;
  479. for (; y < srch; y++)
  480. {
  481. #if __ARM_NEON
  482. dst0 -= 15;
  483. int nn = srcw >> 4;
  484. int remain = srcw - (nn << 4);
  485. #if !NCNN_GNU_INLINE_ASM || __aarch64__
  486. for (; nn > 0; nn--)
  487. {
  488. uint8x8_t _src = vld1_u8(src0);
  489. uint8x8_t _src2 = vld1_u8(src0 + 8);
  490. _src = vrev64_u8(_src);
  491. _src2 = vrev64_u8(_src2);
  492. vst1_u8(dst0, _src2);
  493. vst1_u8(dst0 + 8, _src);
  494. src0 += 16;
  495. dst0 -= 16;
  496. }
  497. #else
  498. if (nn > 0)
  499. {
  500. asm volatile(
  501. "mov r4, #-16 \n"
  502. "0: \n"
  503. "pld [%1, #128] \n"
  504. "vld1.u8 {d0-d1}, [%1]! \n"
  505. "vrev64.u8 d3, d0 \n"
  506. "vrev64.u8 d2, d1 \n"
  507. "subs %0, #1 \n"
  508. "vst1.u8 {d2-d3}, [%2], r4 \n"
  509. "bne 0b \n"
  510. : "=r"(nn), // %0
  511. "=r"(src0), // %1
  512. "=r"(dst0) // %2
  513. : "0"(nn),
  514. "1"(src0),
  515. "2"(dst0)
  516. : "cc", "memory", "q0", "q1", "r4");
  517. }
  518. #endif // __aarch64__
  519. dst0 += 15;
  520. #else
  521. int remain = srcw;
  522. #endif // __ARM_NEON
  523. for (; remain > 0; remain--)
  524. {
  525. *dst0 = *src0;
  526. src0 += 1;
  527. dst0 -= 1;
  528. }
  529. src0 += srcwgap;
  530. dst0 += wgap;
  531. }
  532. }
  533. static void kanna_rotate_2_c2(const unsigned char* src, int srcw, int srch, int srcstride, unsigned char* dst, int w, int /*h*/, int stride)
  534. {
  535. const int srcwgap = srcstride - srcw * 2;
  536. const int wgap = stride + w * 2;
  537. const unsigned char* src0 = src;
  538. unsigned char* dst0 = dst + w * 2 - 2;
  539. int y = 0;
  540. for (; y < srch; y++)
  541. {
  542. #if __ARM_NEON
  543. dst0 -= 7 * 2;
  544. int nn = srcw >> 4;
  545. int remain = srcw - (nn << 4);
  546. #if !NCNN_GNU_INLINE_ASM || __aarch64__
  547. for (; nn > 0; nn--)
  548. {
  549. uint8x8x2_t _src = vld2_u8(src0);
  550. uint8x8x2_t _src2 = vld2_u8(src0 + 8 * 2);
  551. _src.val[0] = vrev64_u8(_src.val[0]);
  552. _src.val[1] = vrev64_u8(_src.val[1]);
  553. _src2.val[0] = vrev64_u8(_src2.val[0]);
  554. _src2.val[1] = vrev64_u8(_src2.val[1]);
  555. vst2_u8(dst0, _src);
  556. vst2_u8(dst0 - 8 * 2, _src2);
  557. src0 += 16 * 2;
  558. dst0 -= 16 * 2;
  559. }
  560. #else
  561. if (nn > 0)
  562. {
  563. asm volatile(
  564. "mov r4, #-16 \n"
  565. "0: \n"
  566. "pld [%1, #128] \n"
  567. "vld2.u8 {d0-d1}, [%1]! \n"
  568. "vrev64.u8 d0, d0 \n"
  569. "pld [%1, #128] \n"
  570. "vld2.u8 {d2-d3}, [%1]! \n"
  571. "vrev64.u8 d1, d1 \n"
  572. "vrev64.u8 d2, d2 \n"
  573. "vst2.u8 {d0-d1}, [%2], r4 \n"
  574. "vrev64.u8 d3, d3 \n"
  575. "subs %0, #1 \n"
  576. "vst2.u8 {d2-d3}, [%2], r4 \n"
  577. "bne 0b \n"
  578. : "=r"(nn), // %0
  579. "=r"(src0), // %1
  580. "=r"(dst0) // %2
  581. : "0"(nn),
  582. "1"(src0),
  583. "2"(dst0)
  584. : "cc", "memory", "q0", "q1", "r4");
  585. }
  586. #endif // __aarch64__
  587. dst0 += 7 * 2;
  588. #else
  589. int remain = srcw;
  590. #endif // __ARM_NEON
  591. for (; remain > 0; remain--)
  592. {
  593. dst0[0] = src0[0];
  594. dst0[1] = src0[1];
  595. src0 += 2;
  596. dst0 -= 2;
  597. }
  598. src0 += srcwgap;
  599. dst0 += wgap;
  600. }
  601. }
  602. static void kanna_rotate_2_c3(const unsigned char* src, int srcw, int srch, int srcstride, unsigned char* dst, int w, int /*h*/, int stride)
  603. {
  604. const int srcwgap = srcstride - srcw * 3;
  605. const int wgap = stride + w * 3;
  606. const unsigned char* src0 = src;
  607. unsigned char* dst0 = dst + w * 3 - 3;
  608. int y = 0;
  609. for (; y < srch; y++)
  610. {
  611. #if __ARM_NEON
  612. dst0 -= 7 * 3;
  613. int nn = srcw >> 4;
  614. int remain = srcw - (nn << 4);
  615. #if !NCNN_GNU_INLINE_ASM || __aarch64__
  616. for (; nn > 0; nn--)
  617. {
  618. uint8x8x3_t _src = vld3_u8(src0);
  619. uint8x8x3_t _src2 = vld3_u8(src0 + 8 * 3);
  620. _src.val[0] = vrev64_u8(_src.val[0]);
  621. _src.val[1] = vrev64_u8(_src.val[1]);
  622. _src.val[2] = vrev64_u8(_src.val[2]);
  623. _src2.val[0] = vrev64_u8(_src2.val[0]);
  624. _src2.val[1] = vrev64_u8(_src2.val[1]);
  625. _src2.val[2] = vrev64_u8(_src2.val[2]);
  626. vst3_u8(dst0, _src);
  627. vst3_u8(dst0 - 8 * 3, _src2);
  628. src0 += 16 * 3;
  629. dst0 -= 16 * 3;
  630. }
  631. #else
  632. if (nn > 0)
  633. {
  634. asm volatile(
  635. "mov r4, #-24 \n"
  636. "0: \n"
  637. "pld [%1, #192] \n"
  638. "vld3.u8 {d0-d2}, [%1]! \n"
  639. "vrev64.u8 d0, d0 \n"
  640. "vrev64.u8 d1, d1 \n"
  641. "pld [%1, #192] \n"
  642. "vld3.u8 {d4-d6}, [%1]! \n"
  643. "vrev64.u8 d2, d2 \n"
  644. "vrev64.u8 d4, d4 \n"
  645. "vst3.u8 {d0-d2}, [%2], r4 \n"
  646. "vrev64.u8 d5, d5 \n"
  647. "vrev64.u8 d6, d6 \n"
  648. "subs %0, #1 \n"
  649. "vst3.u8 {d4-d6}, [%2], r4 \n"
  650. "bne 0b \n"
  651. : "=r"(nn), // %0
  652. "=r"(src0), // %1
  653. "=r"(dst0) // %2
  654. : "0"(nn),
  655. "1"(src0),
  656. "2"(dst0)
  657. : "cc", "memory", "q0", "q1", "q2", "q3", "r4");
  658. }
  659. #endif // __aarch64__
  660. dst0 += 7 * 3;
  661. #else
  662. int remain = srcw;
  663. #endif // __ARM_NEON
  664. for (; remain > 0; remain--)
  665. {
  666. dst0[0] = src0[0];
  667. dst0[1] = src0[1];
  668. dst0[2] = src0[2];
  669. src0 += 3;
  670. dst0 -= 3;
  671. }
  672. src0 += srcwgap;
  673. dst0 += wgap;
  674. }
  675. }
  676. static void kanna_rotate_2_c4(const unsigned char* src, int srcw, int srch, int srcstride, unsigned char* dst, int w, int /*h*/, int stride)
  677. {
  678. const int srcwgap = srcstride - srcw * 4;
  679. const int wgap = stride + w * 4;
  680. const unsigned char* src0 = src;
  681. unsigned char* dst0 = dst + w * 4 - 4;
  682. int y = 0;
  683. for (; y < srch; y++)
  684. {
  685. #if __ARM_NEON
  686. dst0 -= 7 * 4;
  687. int nn = srcw >> 4;
  688. int remain = srcw - (nn << 4);
  689. #if !NCNN_GNU_INLINE_ASM || __aarch64__
  690. for (; nn > 0; nn--)
  691. {
  692. uint8x8x4_t _src = vld4_u8(src0);
  693. uint8x8x4_t _src2 = vld4_u8(src0 + 8 * 4);
  694. _src.val[0] = vrev64_u8(_src.val[0]);
  695. _src.val[1] = vrev64_u8(_src.val[1]);
  696. _src.val[2] = vrev64_u8(_src.val[2]);
  697. _src.val[3] = vrev64_u8(_src.val[3]);
  698. _src2.val[0] = vrev64_u8(_src2.val[0]);
  699. _src2.val[1] = vrev64_u8(_src2.val[1]);
  700. _src2.val[2] = vrev64_u8(_src2.val[2]);
  701. _src2.val[3] = vrev64_u8(_src2.val[3]);
  702. vst4_u8(dst0, _src);
  703. vst4_u8(dst0 - 8 * 4, _src2);
  704. src0 += 16 * 4;
  705. dst0 -= 16 * 4;
  706. }
  707. #else
  708. if (nn > 0)
  709. {
  710. asm volatile(
  711. "mov r4, #-32 \n"
  712. "0: \n"
  713. "pld [%1, #256] \n"
  714. "vld4.u8 {d0-d3}, [%1]! \n"
  715. "vrev64.u8 d0, d0 \n"
  716. "vrev64.u8 d1, d1 \n"
  717. "vrev64.u8 d2, d2 \n"
  718. "pld [%1, #256] \n"
  719. "vld4.u8 {d4-d7}, [%1]! \n"
  720. "vrev64.u8 d3, d3 \n"
  721. "vrev64.u8 d4, d4 \n"
  722. "vrev64.u8 d5, d5 \n"
  723. "vst4.u8 {d0-d3}, [%2], r4 \n"
  724. "vrev64.u8 d6, d6 \n"
  725. "vrev64.u8 d7, d7 \n"
  726. "subs %0, #1 \n"
  727. "vst4.u8 {d4-d7}, [%2], r4 \n"
  728. "bne 0b \n"
  729. : "=r"(nn), // %0
  730. "=r"(src0), // %1
  731. "=r"(dst0) // %2
  732. : "0"(nn),
  733. "1"(src0),
  734. "2"(dst0)
  735. : "cc", "memory", "q0", "q1", "q2", "q3", "r4");
  736. }
  737. #endif // __aarch64__
  738. dst0 += 7 * 4;
  739. #else
  740. int remain = srcw;
  741. #endif // __ARM_NEON
  742. for (; remain > 0; remain--)
  743. {
  744. dst0[0] = src0[0];
  745. dst0[1] = src0[1];
  746. dst0[2] = src0[2];
  747. dst0[3] = src0[3];
  748. src0 += 4;
  749. dst0 -= 4;
  750. }
  751. src0 += srcwgap;
  752. dst0 += wgap;
  753. }
  754. }
  755. static void kanna_rotate_3_c1(const unsigned char* src, int srcw, int srch, int srcstride, unsigned char* dst, int w, int h, int stride)
  756. {
  757. const int srcwgap = srcstride - srcw;
  758. const int wgap = stride - w;
  759. // point to the last dst pixel
  760. unsigned char* dstend = dst + stride * h - wgap;
  761. const unsigned char* src0 = src;
  762. unsigned char* dst0 = dstend - 1;
  763. int y = 0;
  764. for (; y < srch; y++)
  765. {
  766. #if __ARM_NEON
  767. dst0 -= 15;
  768. int nn = srcw >> 4;
  769. int remain = srcw - (nn << 4);
  770. #if !NCNN_GNU_INLINE_ASM || __aarch64__
  771. for (; nn > 0; nn--)
  772. {
  773. uint8x8_t _src = vld1_u8(src0);
  774. uint8x8_t _src2 = vld1_u8(src0 + 8);
  775. _src = vrev64_u8(_src);
  776. _src2 = vrev64_u8(_src2);
  777. vst1_u8(dst0, _src2);
  778. vst1_u8(dst0 + 8, _src);
  779. src0 += 16;
  780. dst0 -= 16;
  781. }
  782. #else
  783. if (nn > 0)
  784. {
  785. asm volatile(
  786. "mov r4, #-16 \n"
  787. "0: \n"
  788. "pld [%1, #128] \n"
  789. "vld1.u8 {d0-d1}, [%1]! \n"
  790. "vrev64.u8 d3, d0 \n"
  791. "vrev64.u8 d2, d1 \n"
  792. "subs %0, #1 \n"
  793. "vst1.u8 {d2-d3}, [%2], r4 \n"
  794. "bne 0b \n"
  795. : "=r"(nn), // %0
  796. "=r"(src0), // %1
  797. "=r"(dst0) // %2
  798. : "0"(nn),
  799. "1"(src0),
  800. "2"(dst0)
  801. : "cc", "memory", "q0", "q1", "r4");
  802. }
  803. #endif // __aarch64__
  804. dst0 += 15;
  805. #else
  806. int remain = srcw;
  807. #endif // __ARM_NEON
  808. for (; remain > 0; remain--)
  809. {
  810. *dst0 = *src0;
  811. src0 += 1;
  812. dst0 -= 1;
  813. }
  814. src0 += srcwgap;
  815. dst0 -= wgap;
  816. }
  817. }
  818. static void kanna_rotate_3_c2(const unsigned char* src, int srcw, int srch, int srcstride, unsigned char* dst, int w, int h, int stride)
  819. {
  820. const int srcwgap = srcstride - srcw * 2;
  821. const int wgap = stride - w * 2;
  822. // point to the last dst pixel
  823. unsigned char* dstend = dst + stride * h - wgap;
  824. const unsigned char* src0 = src;
  825. unsigned char* dst0 = dstend - 2;
  826. int y = 0;
  827. for (; y < srch; y++)
  828. {
  829. #if __ARM_NEON
  830. dst0 -= 7 * 2;
  831. int nn = srcw >> 4;
  832. int remain = srcw - (nn << 4);
  833. #if !NCNN_GNU_INLINE_ASM || __aarch64__
  834. for (; nn > 0; nn--)
  835. {
  836. uint8x8x2_t _src = vld2_u8(src0);
  837. uint8x8x2_t _src2 = vld2_u8(src0 + 8 * 2);
  838. _src.val[0] = vrev64_u8(_src.val[0]);
  839. _src.val[1] = vrev64_u8(_src.val[1]);
  840. _src2.val[0] = vrev64_u8(_src2.val[0]);
  841. _src2.val[1] = vrev64_u8(_src2.val[1]);
  842. vst2_u8(dst0, _src);
  843. vst2_u8(dst0 - 8 * 2, _src2);
  844. src0 += 16 * 2;
  845. dst0 -= 16 * 2;
  846. }
  847. #else
  848. if (nn > 0)
  849. {
  850. asm volatile(
  851. "mov r4, #-16 \n"
  852. "0: \n"
  853. "pld [%1, #128] \n"
  854. "vld2.u8 {d0-d1}, [%1]! \n"
  855. "vrev64.u8 d0, d0 \n"
  856. "pld [%1, #128] \n"
  857. "vld2.u8 {d2-d3}, [%1]! \n"
  858. "vrev64.u8 d1, d1 \n"
  859. "vrev64.u8 d2, d2 \n"
  860. "vst2.u8 {d0-d1}, [%2], r4 \n"
  861. "vrev64.u8 d3, d3 \n"
  862. "subs %0, #1 \n"
  863. "vst2.u8 {d2-d3}, [%2], r4 \n"
  864. "bne 0b \n"
  865. : "=r"(nn), // %0
  866. "=r"(src0), // %1
  867. "=r"(dst0) // %2
  868. : "0"(nn),
  869. "1"(src0),
  870. "2"(dst0)
  871. : "cc", "memory", "q0", "q1", "r4");
  872. }
  873. #endif // __aarch64__
  874. dst0 += 7 * 2;
  875. #else
  876. int remain = srcw;
  877. #endif // __ARM_NEON
  878. for (; remain > 0; remain--)
  879. {
  880. dst0[0] = src0[0];
  881. dst0[1] = src0[1];
  882. src0 += 2;
  883. dst0 -= 2;
  884. }
  885. src0 += srcwgap;
  886. dst0 -= wgap;
  887. }
  888. }
  889. static void kanna_rotate_3_c3(const unsigned char* src, int srcw, int srch, int srcstride, unsigned char* dst, int w, int h, int stride)
  890. {
  891. const int srcwgap = srcstride - srcw * 3;
  892. const int wgap = stride - w * 3;
  893. // point to the last dst pixel
  894. unsigned char* dstend = dst + stride * h - wgap;
  895. const unsigned char* src0 = src;
  896. unsigned char* dst0 = dstend - 3;
  897. int y = 0;
  898. for (; y < srch; y++)
  899. {
  900. #if __ARM_NEON
  901. dst0 -= 7 * 3;
  902. int nn = srcw >> 4;
  903. int remain = srcw - (nn << 4);
  904. #if !NCNN_GNU_INLINE_ASM || __aarch64__
  905. for (; nn > 0; nn--)
  906. {
  907. uint8x8x3_t _src = vld3_u8(src0);
  908. uint8x8x3_t _src2 = vld3_u8(src0 + 8 * 3);
  909. _src.val[0] = vrev64_u8(_src.val[0]);
  910. _src.val[1] = vrev64_u8(_src.val[1]);
  911. _src.val[2] = vrev64_u8(_src.val[2]);
  912. _src2.val[0] = vrev64_u8(_src2.val[0]);
  913. _src2.val[1] = vrev64_u8(_src2.val[1]);
  914. _src2.val[2] = vrev64_u8(_src2.val[2]);
  915. vst3_u8(dst0, _src);
  916. vst3_u8(dst0 - 8 * 3, _src2);
  917. src0 += 16 * 3;
  918. dst0 -= 16 * 3;
  919. }
  920. #else
  921. if (nn > 0)
  922. {
  923. asm volatile(
  924. "mov r4, #-24 \n"
  925. "0: \n"
  926. "pld [%1, #192] \n"
  927. "vld3.u8 {d0-d2}, [%1]! \n"
  928. "vrev64.u8 d0, d0 \n"
  929. "vrev64.u8 d1, d1 \n"
  930. "pld [%1, #192] \n"
  931. "vld3.u8 {d4-d6}, [%1]! \n"
  932. "vrev64.u8 d2, d2 \n"
  933. "vrev64.u8 d4, d4 \n"
  934. "vst3.u8 {d0-d2}, [%2], r4 \n"
  935. "vrev64.u8 d5, d5 \n"
  936. "vrev64.u8 d6, d6 \n"
  937. "subs %0, #1 \n"
  938. "vst3.u8 {d4-d6}, [%2], r4 \n"
  939. "bne 0b \n"
  940. : "=r"(nn), // %0
  941. "=r"(src0), // %1
  942. "=r"(dst0) // %2
  943. : "0"(nn),
  944. "1"(src0),
  945. "2"(dst0)
  946. : "cc", "memory", "q0", "q1", "q2", "q3", "r4");
  947. }
  948. #endif // __aarch64__
  949. dst0 += 7 * 3;
  950. #else
  951. int remain = srcw;
  952. #endif // __ARM_NEON
  953. for (; remain > 0; remain--)
  954. {
  955. dst0[0] = src0[0];
  956. dst0[1] = src0[1];
  957. dst0[2] = src0[2];
  958. src0 += 3;
  959. dst0 -= 3;
  960. }
  961. src0 += srcwgap;
  962. dst0 -= wgap;
  963. }
  964. }
  965. static void kanna_rotate_3_c4(const unsigned char* src, int srcw, int srch, int srcstride, unsigned char* dst, int w, int h, int stride)
  966. {
  967. const int srcwgap = srcstride - srcw * 4;
  968. const int wgap = stride - w * 4;
  969. // point to the last dst pixel
  970. unsigned char* dstend = dst + stride * h - wgap;
  971. const unsigned char* src0 = src;
  972. unsigned char* dst0 = dstend - 4;
  973. int y = 0;
  974. for (; y < srch; y++)
  975. {
  976. #if __ARM_NEON
  977. dst0 -= 7 * 4;
  978. int nn = srcw >> 4;
  979. int remain = srcw - (nn << 4);
  980. #if !NCNN_GNU_INLINE_ASM || __aarch64__
  981. for (; nn > 0; nn--)
  982. {
  983. uint8x8x4_t _src = vld4_u8(src0);
  984. uint8x8x4_t _src2 = vld4_u8(src0 + 8 * 4);
  985. _src.val[0] = vrev64_u8(_src.val[0]);
  986. _src.val[1] = vrev64_u8(_src.val[1]);
  987. _src.val[2] = vrev64_u8(_src.val[2]);
  988. _src.val[3] = vrev64_u8(_src.val[3]);
  989. _src2.val[0] = vrev64_u8(_src2.val[0]);
  990. _src2.val[1] = vrev64_u8(_src2.val[1]);
  991. _src2.val[2] = vrev64_u8(_src2.val[2]);
  992. _src2.val[3] = vrev64_u8(_src2.val[3]);
  993. vst4_u8(dst0, _src);
  994. vst4_u8(dst0 - 8 * 4, _src2);
  995. src0 += 16 * 4;
  996. dst0 -= 16 * 4;
  997. }
  998. #else
  999. if (nn > 0)
  1000. {
  1001. asm volatile(
  1002. "mov r4, #-32 \n"
  1003. "0: \n"
  1004. "pld [%1, #256] \n"
  1005. "vld4.u8 {d0-d3}, [%1]! \n"
  1006. "vrev64.u8 d0, d0 \n"
  1007. "vrev64.u8 d1, d1 \n"
  1008. "vrev64.u8 d2, d2 \n"
  1009. "pld [%1, #256] \n"
  1010. "vld4.u8 {d4-d7}, [%1]! \n"
  1011. "vrev64.u8 d3, d3 \n"
  1012. "vrev64.u8 d4, d4 \n"
  1013. "vrev64.u8 d5, d5 \n"
  1014. "vst4.u8 {d0-d3}, [%2], r4 \n"
  1015. "vrev64.u8 d6, d6 \n"
  1016. "vrev64.u8 d7, d7 \n"
  1017. "subs %0, #1 \n"
  1018. "vst4.u8 {d4-d7}, [%2], r4 \n"
  1019. "bne 0b \n"
  1020. : "=r"(nn), // %0
  1021. "=r"(src0), // %1
  1022. "=r"(dst0) // %2
  1023. : "0"(nn),
  1024. "1"(src0),
  1025. "2"(dst0)
  1026. : "cc", "memory", "q0", "q1", "q2", "q3", "r4");
  1027. }
  1028. #endif // __aarch64__
  1029. dst0 += 7 * 4;
  1030. #else
  1031. int remain = srcw;
  1032. #endif // __ARM_NEON
  1033. for (; remain > 0; remain--)
  1034. {
  1035. dst0[0] = src0[0];
  1036. dst0[1] = src0[1];
  1037. dst0[2] = src0[2];
  1038. dst0[3] = src0[3];
  1039. src0 += 4;
  1040. dst0 -= 4;
  1041. }
  1042. src0 += srcwgap;
  1043. dst0 -= wgap;
  1044. }
  1045. }
  1046. static void kanna_rotate_4_c1(const unsigned char* src, int srcw, int srch, int srcstride, unsigned char* dst, int w, int h, int stride)
  1047. {
  1048. const int srcwgap = srcstride - srcw;
  1049. const int wgap = stride + w;
  1050. // point to the last dst pixel row
  1051. unsigned char* dstend = dst + stride * (h - 1);
  1052. const unsigned char* src0 = src;
  1053. const unsigned char* src1 = src + srcstride;
  1054. unsigned char* dst0 = dstend;
  1055. unsigned char* dst1 = dstend - stride;
  1056. int y = 0;
  1057. for (; y + 1 < srch; y += 2)
  1058. {
  1059. #if __ARM_NEON
  1060. int nn = srcw >> 5;
  1061. int remain = srcw - (nn << 5);
  1062. #if !NCNN_GNU_INLINE_ASM || __aarch64__
  1063. for (; nn > 0; nn--)
  1064. {
  1065. uint8x16_t _src0 = vld1q_u8(src0);
  1066. uint8x16_t _src0n = vld1q_u8(src0 + 16);
  1067. vst1q_u8(dst0, _src0);
  1068. vst1q_u8(dst0 + 16, _src0n);
  1069. uint8x16_t _src1 = vld1q_u8(src1);
  1070. uint8x16_t _src1n = vld1q_u8(src1 + 16);
  1071. vst1q_u8(dst1, _src1);
  1072. vst1q_u8(dst1 + 16, _src1n);
  1073. src0 += 32;
  1074. src1 += 32;
  1075. dst0 += 32;
  1076. dst1 += 32;
  1077. }
  1078. #else
  1079. if (nn > 0)
  1080. {
  1081. asm volatile(
  1082. "0: \n"
  1083. "pld [%1, #256] \n"
  1084. "vld1.u8 {d0-d3}, [%1]! \n"
  1085. "pld [%2, #256] \n"
  1086. "vld1.u8 {d4-d7}, [%2]! \n"
  1087. "subs %0, #1 \n"
  1088. "vst1.u8 {d0-d3}, [%3]! \n"
  1089. "vst1.u8 {d4-d7}, [%4]! \n"
  1090. "bne 0b \n"
  1091. : "=r"(nn), // %0
  1092. "=r"(src0), // %1
  1093. "=r"(src1), // %2
  1094. "=r"(dst0), // %3
  1095. "=r"(dst1) // %4
  1096. : "0"(nn),
  1097. "1"(src0),
  1098. "2"(src1),
  1099. "3"(dst0),
  1100. "4"(dst1)
  1101. : "cc", "memory", "q0", "q1", "q2", "q3");
  1102. }
  1103. #endif // __aarch64__
  1104. #else
  1105. int remain = srcw;
  1106. #endif // __ARM_NEON
  1107. for (; remain > 0; remain--)
  1108. {
  1109. *dst0++ = *src0++;
  1110. *dst1++ = *src1++;
  1111. }
  1112. src0 += srcwgap + srcstride;
  1113. src1 += srcwgap + srcstride;
  1114. dst0 -= wgap + stride;
  1115. dst1 -= wgap + stride;
  1116. }
  1117. for (; y < srch; y++)
  1118. {
  1119. #if __ARM_NEON
  1120. int nn = srcw >> 5;
  1121. int remain = srcw - (nn << 5);
  1122. #if !NCNN_GNU_INLINE_ASM || __aarch64__
  1123. for (; nn > 0; nn--)
  1124. {
  1125. uint8x16_t _src = vld1q_u8(src0);
  1126. uint8x16_t _src2 = vld1q_u8(src0 + 16);
  1127. vst1q_u8(dst0, _src);
  1128. vst1q_u8(dst0 + 16, _src2);
  1129. src0 += 32;
  1130. dst0 += 32;
  1131. }
  1132. #else
  1133. if (nn > 0)
  1134. {
  1135. asm volatile(
  1136. "0: \n"
  1137. "pld [%1, #256] \n"
  1138. "vld1.u8 {d0-d3}, [%1]! \n"
  1139. "subs %0, #1 \n"
  1140. "vst1.u8 {d0-d3}, [%2]! \n"
  1141. "bne 0b \n"
  1142. : "=r"(nn), // %0
  1143. "=r"(src0), // %1
  1144. "=r"(dst0) // %2
  1145. : "0"(nn),
  1146. "1"(src0),
  1147. "2"(dst0)
  1148. : "cc", "memory", "q0", "q1");
  1149. }
  1150. #endif // __aarch64__
  1151. #else
  1152. int remain = srcw;
  1153. #endif // __ARM_NEON
  1154. for (; remain > 0; remain--)
  1155. {
  1156. *dst0++ = *src0++;
  1157. }
  1158. src0 += srcwgap;
  1159. dst0 -= wgap;
  1160. }
  1161. }
  1162. static void kanna_rotate_4_c2(const unsigned char* src, int srcw, int srch, int srcstride, unsigned char* dst, int w, int h, int stride)
  1163. {
  1164. const int srcwgap = srcstride - srcw * 2;
  1165. const int wgap = stride + w * 2;
  1166. // point to the last dst pixel row
  1167. unsigned char* dstend = dst + stride * (h - 1);
  1168. int size = srcw * 2;
  1169. const unsigned char* src0 = src;
  1170. const unsigned char* src1 = src + srcstride;
  1171. unsigned char* dst0 = dstend;
  1172. unsigned char* dst1 = dstend - stride;
  1173. int y = 0;
  1174. for (; y + 1 < srch; y += 2)
  1175. {
  1176. #if __ARM_NEON
  1177. int nn = size >> 5;
  1178. int remain = size - (nn << 5);
  1179. #if !NCNN_GNU_INLINE_ASM || __aarch64__
  1180. for (; nn > 0; nn--)
  1181. {
  1182. uint8x16_t _src0 = vld1q_u8(src0);
  1183. uint8x16_t _src0n = vld1q_u8(src0 + 16);
  1184. vst1q_u8(dst0, _src0);
  1185. vst1q_u8(dst0 + 16, _src0n);
  1186. uint8x16_t _src1 = vld1q_u8(src1);
  1187. uint8x16_t _src1n = vld1q_u8(src1 + 16);
  1188. vst1q_u8(dst1, _src1);
  1189. vst1q_u8(dst1 + 16, _src1n);
  1190. src0 += 32;
  1191. src1 += 32;
  1192. dst0 += 32;
  1193. dst1 += 32;
  1194. }
  1195. #else
  1196. if (nn > 0)
  1197. {
  1198. asm volatile(
  1199. "0: \n"
  1200. "pld [%1, #256] \n"
  1201. "vld1.u8 {d0-d3}, [%1]! \n"
  1202. "pld [%2, #256] \n"
  1203. "vld1.u8 {d4-d7}, [%2]! \n"
  1204. "subs %0, #1 \n"
  1205. "vst1.u8 {d0-d3}, [%3]! \n"
  1206. "vst1.u8 {d4-d7}, [%4]! \n"
  1207. "bne 0b \n"
  1208. : "=r"(nn), // %0
  1209. "=r"(src0), // %1
  1210. "=r"(src1), // %2
  1211. "=r"(dst0), // %3
  1212. "=r"(dst1) // %4
  1213. : "0"(nn),
  1214. "1"(src0),
  1215. "2"(src1),
  1216. "3"(dst0),
  1217. "4"(dst1)
  1218. : "cc", "memory", "q0", "q1", "q2", "q3");
  1219. }
  1220. #endif // __aarch64__
  1221. #else
  1222. int remain = size;
  1223. #endif // __ARM_NEON
  1224. for (; remain > 0; remain--)
  1225. {
  1226. *dst0++ = *src0++;
  1227. *dst1++ = *src1++;
  1228. }
  1229. src0 += srcwgap + srcstride;
  1230. src1 += srcwgap + srcstride;
  1231. dst0 -= wgap + stride;
  1232. dst1 -= wgap + stride;
  1233. }
  1234. for (; y < srch; y++)
  1235. {
  1236. #if __ARM_NEON
  1237. int nn = size >> 5;
  1238. int remain = size - (nn << 5);
  1239. #if !NCNN_GNU_INLINE_ASM || __aarch64__
  1240. for (; nn > 0; nn--)
  1241. {
  1242. uint8x16_t _src = vld1q_u8(src0);
  1243. uint8x16_t _src2 = vld1q_u8(src0 + 16);
  1244. vst1q_u8(dst0, _src);
  1245. vst1q_u8(dst0 + 16, _src2);
  1246. src0 += 32;
  1247. dst0 += 32;
  1248. }
  1249. #else
  1250. if (nn > 0)
  1251. {
  1252. asm volatile(
  1253. "0: \n"
  1254. "pld [%1, #256] \n"
  1255. "vld1.u8 {d0-d3}, [%1]! \n"
  1256. "subs %0, #1 \n"
  1257. "vst1.u8 {d0-d3}, [%2]! \n"
  1258. "bne 0b \n"
  1259. : "=r"(nn), // %0
  1260. "=r"(src0), // %1
  1261. "=r"(dst0) // %2
  1262. : "0"(nn),
  1263. "1"(src0),
  1264. "2"(dst0)
  1265. : "cc", "memory", "q0", "q1");
  1266. }
  1267. #endif // __aarch64__
  1268. #else
  1269. int remain = size;
  1270. #endif // __ARM_NEON
  1271. for (; remain > 0; remain--)
  1272. {
  1273. *dst0++ = *src0++;
  1274. }
  1275. src0 += srcwgap;
  1276. dst0 -= wgap;
  1277. }
  1278. }
  1279. static void kanna_rotate_4_c3(const unsigned char* src, int srcw, int srch, int srcstride, unsigned char* dst, int w, int h, int stride)
  1280. {
  1281. const int srcwgap = srcstride - srcw * 3;
  1282. const int wgap = stride + w * 3;
  1283. // point to the last dst pixel row
  1284. unsigned char* dstend = dst + stride * (h - 1);
  1285. int size = srcw * 3;
  1286. const unsigned char* src0 = src;
  1287. const unsigned char* src1 = src + srcstride;
  1288. unsigned char* dst0 = dstend;
  1289. unsigned char* dst1 = dstend - stride;
  1290. int y = 0;
  1291. for (; y + 1 < srch; y += 2)
  1292. {
  1293. #if __ARM_NEON
  1294. int nn = size >> 5;
  1295. int remain = size - (nn << 5);
  1296. #if !NCNN_GNU_INLINE_ASM || __aarch64__
  1297. for (; nn > 0; nn--)
  1298. {
  1299. uint8x16_t _src0 = vld1q_u8(src0);
  1300. uint8x16_t _src0n = vld1q_u8(src0 + 16);
  1301. vst1q_u8(dst0, _src0);
  1302. vst1q_u8(dst0 + 16, _src0n);
  1303. uint8x16_t _src1 = vld1q_u8(src1);
  1304. uint8x16_t _src1n = vld1q_u8(src1 + 16);
  1305. vst1q_u8(dst1, _src1);
  1306. vst1q_u8(dst1 + 16, _src1n);
  1307. src0 += 32;
  1308. src1 += 32;
  1309. dst0 += 32;
  1310. dst1 += 32;
  1311. }
  1312. #else
  1313. if (nn > 0)
  1314. {
  1315. asm volatile(
  1316. "0: \n"
  1317. "pld [%1, #256] \n"
  1318. "vld1.u8 {d0-d3}, [%1]! \n"
  1319. "pld [%2, #256] \n"
  1320. "vld1.u8 {d4-d7}, [%2]! \n"
  1321. "subs %0, #1 \n"
  1322. "vst1.u8 {d0-d3}, [%3]! \n"
  1323. "vst1.u8 {d4-d7}, [%4]! \n"
  1324. "bne 0b \n"
  1325. : "=r"(nn), // %0
  1326. "=r"(src0), // %1
  1327. "=r"(src1), // %2
  1328. "=r"(dst0), // %3
  1329. "=r"(dst1) // %4
  1330. : "0"(nn),
  1331. "1"(src0),
  1332. "2"(src1),
  1333. "3"(dst0),
  1334. "4"(dst1)
  1335. : "cc", "memory", "q0", "q1", "q2", "q3");
  1336. }
  1337. #endif // __aarch64__
  1338. #else
  1339. int remain = size;
  1340. #endif // __ARM_NEON
  1341. for (; remain > 0; remain--)
  1342. {
  1343. *dst0++ = *src0++;
  1344. *dst1++ = *src1++;
  1345. }
  1346. src0 += srcwgap + srcstride;
  1347. src1 += srcwgap + srcstride;
  1348. dst0 -= wgap + stride;
  1349. dst1 -= wgap + stride;
  1350. }
  1351. for (; y < srch; y++)
  1352. {
  1353. #if __ARM_NEON
  1354. int nn = size >> 5;
  1355. int remain = size - (nn << 5);
  1356. #if !NCNN_GNU_INLINE_ASM || __aarch64__
  1357. for (; nn > 0; nn--)
  1358. {
  1359. uint8x16_t _src = vld1q_u8(src0);
  1360. uint8x16_t _src2 = vld1q_u8(src0 + 16);
  1361. vst1q_u8(dst0, _src);
  1362. vst1q_u8(dst0 + 16, _src2);
  1363. src0 += 32;
  1364. dst0 += 32;
  1365. }
  1366. #else
  1367. if (nn > 0)
  1368. {
  1369. asm volatile(
  1370. "0: \n"
  1371. "pld [%1, #256] \n"
  1372. "vld1.u8 {d0-d3}, [%1]! \n"
  1373. "subs %0, #1 \n"
  1374. "vst1.u8 {d0-d3}, [%2]! \n"
  1375. "bne 0b \n"
  1376. : "=r"(nn), // %0
  1377. "=r"(src0), // %1
  1378. "=r"(dst0) // %2
  1379. : "0"(nn),
  1380. "1"(src0),
  1381. "2"(dst0)
  1382. : "cc", "memory", "q0", "q1");
  1383. }
  1384. #endif // __aarch64__
  1385. #else
  1386. int remain = size;
  1387. #endif // __ARM_NEON
  1388. for (; remain > 0; remain--)
  1389. {
  1390. *dst0++ = *src0++;
  1391. }
  1392. src0 += srcwgap;
  1393. dst0 -= wgap;
  1394. }
  1395. }
  1396. static void kanna_rotate_4_c4(const unsigned char* src, int srcw, int srch, int srcstride, unsigned char* dst, int w, int h, int stride)
  1397. {
  1398. const int srcwgap = srcstride - srcw * 4;
  1399. const int wgap = stride + w * 4;
  1400. // point to the last dst pixel row
  1401. unsigned char* dstend = dst + stride * (h - 1);
  1402. int size = srcw * 4;
  1403. const unsigned char* src0 = src;
  1404. const unsigned char* src1 = src + srcstride;
  1405. unsigned char* dst0 = dstend;
  1406. unsigned char* dst1 = dstend - stride;
  1407. int y = 0;
  1408. for (; y + 1 < srch; y += 2)
  1409. {
  1410. #if __ARM_NEON
  1411. int nn = size >> 5;
  1412. int remain = size - (nn << 5);
  1413. #if !NCNN_GNU_INLINE_ASM || __aarch64__
  1414. for (; nn > 0; nn--)
  1415. {
  1416. uint8x16_t _src0 = vld1q_u8(src0);
  1417. uint8x16_t _src0n = vld1q_u8(src0 + 16);
  1418. vst1q_u8(dst0, _src0);
  1419. vst1q_u8(dst0 + 16, _src0n);
  1420. uint8x16_t _src1 = vld1q_u8(src1);
  1421. uint8x16_t _src1n = vld1q_u8(src1 + 16);
  1422. vst1q_u8(dst1, _src1);
  1423. vst1q_u8(dst1 + 16, _src1n);
  1424. src0 += 32;
  1425. src1 += 32;
  1426. dst0 += 32;
  1427. dst1 += 32;
  1428. }
  1429. #else
  1430. if (nn > 0)
  1431. {
  1432. asm volatile(
  1433. "0: \n"
  1434. "pld [%1, #256] \n"
  1435. "vld1.u8 {d0-d3}, [%1]! \n"
  1436. "pld [%2, #256] \n"
  1437. "vld1.u8 {d4-d7}, [%2]! \n"
  1438. "subs %0, #1 \n"
  1439. "vst1.u8 {d0-d3}, [%3]! \n"
  1440. "vst1.u8 {d4-d7}, [%4]! \n"
  1441. "bne 0b \n"
  1442. : "=r"(nn), // %0
  1443. "=r"(src0), // %1
  1444. "=r"(src1), // %2
  1445. "=r"(dst0), // %3
  1446. "=r"(dst1) // %4
  1447. : "0"(nn),
  1448. "1"(src0),
  1449. "2"(src1),
  1450. "3"(dst0),
  1451. "4"(dst1)
  1452. : "cc", "memory", "q0", "q1", "q2", "q3");
  1453. }
  1454. #endif // __aarch64__
  1455. #else
  1456. int remain = size;
  1457. #endif // __ARM_NEON
  1458. for (; remain > 0; remain--)
  1459. {
  1460. *dst0++ = *src0++;
  1461. *dst1++ = *src1++;
  1462. }
  1463. src0 += srcwgap + srcstride;
  1464. src1 += srcwgap + srcstride;
  1465. dst0 -= wgap + stride;
  1466. dst1 -= wgap + stride;
  1467. }
  1468. for (; y < srch; y++)
  1469. {
  1470. #if __ARM_NEON
  1471. int nn = size >> 5;
  1472. int remain = size - (nn << 5);
  1473. #if !NCNN_GNU_INLINE_ASM || __aarch64__
  1474. for (; nn > 0; nn--)
  1475. {
  1476. uint8x16_t _src = vld1q_u8(src0);
  1477. uint8x16_t _src2 = vld1q_u8(src0 + 16);
  1478. vst1q_u8(dst0, _src);
  1479. vst1q_u8(dst0 + 16, _src2);
  1480. src0 += 32;
  1481. dst0 += 32;
  1482. }
  1483. #else
  1484. if (nn > 0)
  1485. {
  1486. asm volatile(
  1487. "0: \n"
  1488. "pld [%1, #256] \n"
  1489. "vld1.u8 {d0-d3}, [%1]! \n"
  1490. "subs %0, #1 \n"
  1491. "vst1.u8 {d0-d3}, [%2]! \n"
  1492. "bne 0b \n"
  1493. : "=r"(nn), // %0
  1494. "=r"(src0), // %1
  1495. "=r"(dst0) // %2
  1496. : "0"(nn),
  1497. "1"(src0),
  1498. "2"(dst0)
  1499. : "cc", "memory", "q0", "q1");
  1500. }
  1501. #endif // __aarch64__
  1502. #else
  1503. int remain = size;
  1504. #endif // __ARM_NEON
  1505. for (; remain > 0; remain--)
  1506. {
  1507. *dst0++ = *src0++;
  1508. }
  1509. src0 += srcwgap;
  1510. dst0 -= wgap;
  1511. }
  1512. }
  1513. static void kanna_rotate_5_c1(const unsigned char* src, int srcw, int srch, int srcstride, unsigned char* dst, int /*w*/, int /*h*/, int stride)
  1514. {
  1515. const int srcwgap = srcstride - srcw;
  1516. const unsigned char* src0 = src;
  1517. int y = 0;
  1518. #if __ARM_NEON
  1519. for (; y + 7 < srch; y += 8)
  1520. {
  1521. const unsigned char* src1 = src0 + srcstride;
  1522. unsigned char* dst0 = dst + y;
  1523. unsigned char* dst1 = dst + y + stride;
  1524. int src_step = 2 * srcstride;
  1525. int dst_step = 2 * stride;
  1526. int nn = srcw >> 3;
  1527. int remain = srcw - (nn << 3);
  1528. #if !NCNN_GNU_INLINE_ASM || __aarch64__
  1529. for (; nn > 0; nn--)
  1530. {
  1531. uint8x8_t _src0 = vld1_u8(src0);
  1532. uint8x8_t _src1 = vld1_u8(src1);
  1533. uint8x8_t _src2 = vld1_u8(src0 + src_step);
  1534. uint8x8_t _src3 = vld1_u8(src1 + src_step);
  1535. uint8x8_t _src4 = vld1_u8(src0 + 2 * src_step);
  1536. uint8x8_t _src5 = vld1_u8(src1 + 2 * src_step);
  1537. uint8x8_t _src6 = vld1_u8(src0 + 3 * src_step);
  1538. uint8x8_t _src7 = vld1_u8(src1 + 3 * src_step);
  1539. uint8x8x2_t _src01t_r = vtrn_u8(_src0, _src1);
  1540. uint8x8x2_t _src23t_r = vtrn_u8(_src2, _src3);
  1541. uint8x8x2_t _src45t_r = vtrn_u8(_src4, _src5);
  1542. uint8x8x2_t _src67t_r = vtrn_u8(_src6, _src7);
  1543. uint16x4x2_t _src02tt_r = vtrn_u16(vreinterpret_u16_u8(_src01t_r.val[0]), vreinterpret_u16_u8(_src23t_r.val[0]));
  1544. uint16x4x2_t _src13tt_r = vtrn_u16(vreinterpret_u16_u8(_src01t_r.val[1]), vreinterpret_u16_u8(_src23t_r.val[1]));
  1545. uint16x4x2_t _src46tt_r = vtrn_u16(vreinterpret_u16_u8(_src45t_r.val[0]), vreinterpret_u16_u8(_src67t_r.val[0]));
  1546. uint16x4x2_t _src57tt_r = vtrn_u16(vreinterpret_u16_u8(_src45t_r.val[1]), vreinterpret_u16_u8(_src67t_r.val[1]));
  1547. uint32x2x2_t _src04ttt_r = vtrn_u32(vreinterpret_u32_u16(_src02tt_r.val[0]), vreinterpret_u32_u16(_src46tt_r.val[0]));
  1548. uint32x2x2_t _src15ttt_r = vtrn_u32(vreinterpret_u32_u16(_src13tt_r.val[0]), vreinterpret_u32_u16(_src57tt_r.val[0]));
  1549. uint32x2x2_t _src26ttt_r = vtrn_u32(vreinterpret_u32_u16(_src02tt_r.val[1]), vreinterpret_u32_u16(_src46tt_r.val[1]));
  1550. uint32x2x2_t _src37ttt_r = vtrn_u32(vreinterpret_u32_u16(_src13tt_r.val[1]), vreinterpret_u32_u16(_src57tt_r.val[1]));
  1551. uint8x8_t _dst0 = vreinterpret_u8_u32(_src04ttt_r.val[0]);
  1552. uint8x8_t _dst1 = vreinterpret_u8_u32(_src15ttt_r.val[0]);
  1553. uint8x8_t _dst2 = vreinterpret_u8_u32(_src26ttt_r.val[0]);
  1554. uint8x8_t _dst3 = vreinterpret_u8_u32(_src37ttt_r.val[0]);
  1555. uint8x8_t _dst4 = vreinterpret_u8_u32(_src04ttt_r.val[1]);
  1556. uint8x8_t _dst5 = vreinterpret_u8_u32(_src15ttt_r.val[1]);
  1557. uint8x8_t _dst6 = vreinterpret_u8_u32(_src26ttt_r.val[1]);
  1558. uint8x8_t _dst7 = vreinterpret_u8_u32(_src37ttt_r.val[1]);
  1559. vst1_u8(dst0, _dst0);
  1560. vst1_u8(dst1, _dst1);
  1561. vst1_u8(dst0 + dst_step, _dst2);
  1562. vst1_u8(dst1 + dst_step, _dst3);
  1563. vst1_u8(dst0 + 2 * dst_step, _dst4);
  1564. vst1_u8(dst1 + 2 * dst_step, _dst5);
  1565. vst1_u8(dst0 + 3 * dst_step, _dst6);
  1566. vst1_u8(dst1 + 3 * dst_step, _dst7);
  1567. src0 += 8;
  1568. src1 += 8;
  1569. dst0 += 4 * dst_step;
  1570. dst1 += 4 * dst_step;
  1571. }
  1572. #else
  1573. if (nn > 0)
  1574. {
  1575. asm volatile(
  1576. "0: \n"
  1577. "pld [%1, #64] \n"
  1578. "vld1.u8 {d0}, [%1], %10 \n"
  1579. "pld [%2, #64] \n"
  1580. "vld1.u8 {d1}, [%2], %10 \n"
  1581. "pld [%1, #64] \n"
  1582. "vld1.u8 {d2}, [%1], %10 \n"
  1583. "vtrn.u8 d0, d1 \n" // _src01t_r
  1584. "pld [%2, #64] \n"
  1585. "vld1.u8 {d3}, [%2], %10 \n"
  1586. "pld [%1, #64] \n"
  1587. "vld1.u8 {d4}, [%1], %10 \n"
  1588. "vtrn.u8 d2, d3 \n" // _src23t_r
  1589. "pld [%2, #64] \n"
  1590. "vld1.u8 {d5}, [%2], %10 \n"
  1591. "pld [%1, #64] \n"
  1592. "vld1.u8 {d6}, [%1], %10 \n"
  1593. "vtrn.u8 d4, d5 \n" // _src45t_r
  1594. "pld [%2, #64] \n"
  1595. "vld1.u8 {d7}, [%2], %10 \n"
  1596. "vtrn.u8 d6, d7 \n" // _src67t_r
  1597. "sub %1, %1, %10, lsl #2 \n" // restore src0
  1598. "vtrn.u16 q0, q1 \n" // _src02tt_r _src13tt_r
  1599. "sub %2, %2, %10, lsl #2 \n" // restore src1
  1600. "vtrn.u16 q2, q3 \n" // _src13tt_r _src46tt_r
  1601. "add %1, #8 \n" // src0 += 8
  1602. "vtrn.u32 q0, q2 \n" // _src04ttt_r _src15ttt_r
  1603. "add %2, #8 \n" // src1 += 8
  1604. "vtrn.u32 q1, q3 \n" // _src26ttt_r _src37ttt_r
  1605. "vst1.u8 {d0}, [%3], %11 \n"
  1606. "vst1.u8 {d1}, [%4], %11 \n"
  1607. "subs %0, #1 \n"
  1608. "vst1.u8 {d2}, [%3], %11 \n"
  1609. "vst1.u8 {d3}, [%4], %11 \n"
  1610. "vst1.u8 {d4}, [%3], %11 \n"
  1611. "vst1.u8 {d5}, [%4], %11 \n"
  1612. "vst1.u8 {d6}, [%3], %11 \n"
  1613. "vst1.u8 {d7}, [%4], %11 \n"
  1614. "bne 0b \n"
  1615. : "=r"(nn), // %0
  1616. "=r"(src0), // %1
  1617. "=r"(src1), // %2
  1618. "=r"(dst0), // %3
  1619. "=r"(dst1) // %4
  1620. : "0"(nn),
  1621. "1"(src0),
  1622. "2"(src1),
  1623. "3"(dst0),
  1624. "4"(dst1),
  1625. "r"(src_step), // %10
  1626. "r"(dst_step) // %11
  1627. : "cc", "memory", "q0", "q1", "q2", "q3");
  1628. }
  1629. #endif // __aarch64__
  1630. for (; remain > 0; remain--)
  1631. {
  1632. dst0[0] = src0[0];
  1633. dst0[1] = src1[0];
  1634. dst0[2] = src0[0 + src_step];
  1635. dst0[3] = src1[0 + src_step];
  1636. dst0[4] = src0[0 + 2 * src_step];
  1637. dst0[5] = src1[0 + 2 * src_step];
  1638. dst0[6] = src0[0 + 3 * src_step];
  1639. dst0[7] = src1[0 + 3 * src_step];
  1640. src0 += 1;
  1641. src1 += 1;
  1642. dst0 += stride;
  1643. }
  1644. src0 += srcwgap + 7 * srcstride;
  1645. }
  1646. #endif // __ARM_NEON
  1647. for (; y < srch; y++)
  1648. {
  1649. unsigned char* dst0 = dst + y;
  1650. int x = 0;
  1651. for (; x < srcw; x++)
  1652. {
  1653. *dst0 = *src0;
  1654. src0 += 1;
  1655. dst0 += stride;
  1656. }
  1657. src0 += srcwgap;
  1658. }
  1659. }
  1660. static void kanna_rotate_5_c2(const unsigned char* src, int srcw, int srch, int srcstride, unsigned char* dst, int /*w*/, int /*h*/, int stride)
  1661. {
  1662. const int srcwgap = srcstride - srcw * 2;
  1663. const unsigned char* src0 = src;
  1664. int y = 0;
  1665. #if __ARM_NEON
  1666. for (; y + 7 < srch; y += 8)
  1667. {
  1668. const unsigned char* src1 = src0 + srcstride;
  1669. unsigned char* dst0 = dst + y * 2;
  1670. unsigned char* dst1 = dst + y * 2 + stride;
  1671. int src_step = 2 * srcstride;
  1672. int dst_step = 2 * stride;
  1673. int nn = srcw >> 3;
  1674. int remain = srcw - (nn << 3);
  1675. #if !NCNN_GNU_INLINE_ASM || __aarch64__
  1676. for (; nn > 0; nn--)
  1677. {
  1678. uint8x8x2_t _src0 = vld2_u8(src0);
  1679. uint8x8x2_t _src1 = vld2_u8(src1);
  1680. uint8x8x2_t _src2 = vld2_u8(src0 + src_step);
  1681. uint8x8x2_t _src3 = vld2_u8(src1 + src_step);
  1682. uint8x8x2_t _src4 = vld2_u8(src0 + 2 * src_step);
  1683. uint8x8x2_t _src5 = vld2_u8(src1 + 2 * src_step);
  1684. uint8x8x2_t _src6 = vld2_u8(src0 + 3 * src_step);
  1685. uint8x8x2_t _src7 = vld2_u8(src1 + 3 * src_step);
  1686. uint8x8x2_t _src01t_r = vtrn_u8(_src0.val[0], _src1.val[0]);
  1687. uint8x8x2_t _src23t_r = vtrn_u8(_src2.val[0], _src3.val[0]);
  1688. uint8x8x2_t _src45t_r = vtrn_u8(_src4.val[0], _src5.val[0]);
  1689. uint8x8x2_t _src67t_r = vtrn_u8(_src6.val[0], _src7.val[0]);
  1690. uint8x8x2_t _src01t_g = vtrn_u8(_src0.val[1], _src1.val[1]);
  1691. uint8x8x2_t _src23t_g = vtrn_u8(_src2.val[1], _src3.val[1]);
  1692. uint8x8x2_t _src45t_g = vtrn_u8(_src4.val[1], _src5.val[1]);
  1693. uint8x8x2_t _src67t_g = vtrn_u8(_src6.val[1], _src7.val[1]);
  1694. uint16x4x2_t _src02tt_r = vtrn_u16(vreinterpret_u16_u8(_src01t_r.val[0]), vreinterpret_u16_u8(_src23t_r.val[0]));
  1695. uint16x4x2_t _src13tt_r = vtrn_u16(vreinterpret_u16_u8(_src01t_r.val[1]), vreinterpret_u16_u8(_src23t_r.val[1]));
  1696. uint16x4x2_t _src46tt_r = vtrn_u16(vreinterpret_u16_u8(_src45t_r.val[0]), vreinterpret_u16_u8(_src67t_r.val[0]));
  1697. uint16x4x2_t _src57tt_r = vtrn_u16(vreinterpret_u16_u8(_src45t_r.val[1]), vreinterpret_u16_u8(_src67t_r.val[1]));
  1698. uint16x4x2_t _src02tt_g = vtrn_u16(vreinterpret_u16_u8(_src01t_g.val[0]), vreinterpret_u16_u8(_src23t_g.val[0]));
  1699. uint16x4x2_t _src13tt_g = vtrn_u16(vreinterpret_u16_u8(_src01t_g.val[1]), vreinterpret_u16_u8(_src23t_g.val[1]));
  1700. uint16x4x2_t _src46tt_g = vtrn_u16(vreinterpret_u16_u8(_src45t_g.val[0]), vreinterpret_u16_u8(_src67t_g.val[0]));
  1701. uint16x4x2_t _src57tt_g = vtrn_u16(vreinterpret_u16_u8(_src45t_g.val[1]), vreinterpret_u16_u8(_src67t_g.val[1]));
  1702. uint32x2x2_t _src04ttt_r = vtrn_u32(vreinterpret_u32_u16(_src02tt_r.val[0]), vreinterpret_u32_u16(_src46tt_r.val[0]));
  1703. uint32x2x2_t _src15ttt_r = vtrn_u32(vreinterpret_u32_u16(_src13tt_r.val[0]), vreinterpret_u32_u16(_src57tt_r.val[0]));
  1704. uint32x2x2_t _src26ttt_r = vtrn_u32(vreinterpret_u32_u16(_src02tt_r.val[1]), vreinterpret_u32_u16(_src46tt_r.val[1]));
  1705. uint32x2x2_t _src37ttt_r = vtrn_u32(vreinterpret_u32_u16(_src13tt_r.val[1]), vreinterpret_u32_u16(_src57tt_r.val[1]));
  1706. uint32x2x2_t _src04ttt_g = vtrn_u32(vreinterpret_u32_u16(_src02tt_g.val[0]), vreinterpret_u32_u16(_src46tt_g.val[0]));
  1707. uint32x2x2_t _src15ttt_g = vtrn_u32(vreinterpret_u32_u16(_src13tt_g.val[0]), vreinterpret_u32_u16(_src57tt_g.val[0]));
  1708. uint32x2x2_t _src26ttt_g = vtrn_u32(vreinterpret_u32_u16(_src02tt_g.val[1]), vreinterpret_u32_u16(_src46tt_g.val[1]));
  1709. uint32x2x2_t _src37ttt_g = vtrn_u32(vreinterpret_u32_u16(_src13tt_g.val[1]), vreinterpret_u32_u16(_src57tt_g.val[1]));
  1710. uint8x8x2_t _dst0;
  1711. uint8x8x2_t _dst1;
  1712. uint8x8x2_t _dst2;
  1713. uint8x8x2_t _dst3;
  1714. uint8x8x2_t _dst4;
  1715. uint8x8x2_t _dst5;
  1716. uint8x8x2_t _dst6;
  1717. uint8x8x2_t _dst7;
  1718. _dst0.val[0] = vreinterpret_u8_u32(_src04ttt_r.val[0]);
  1719. _dst1.val[0] = vreinterpret_u8_u32(_src15ttt_r.val[0]);
  1720. _dst2.val[0] = vreinterpret_u8_u32(_src26ttt_r.val[0]);
  1721. _dst3.val[0] = vreinterpret_u8_u32(_src37ttt_r.val[0]);
  1722. _dst4.val[0] = vreinterpret_u8_u32(_src04ttt_r.val[1]);
  1723. _dst5.val[0] = vreinterpret_u8_u32(_src15ttt_r.val[1]);
  1724. _dst6.val[0] = vreinterpret_u8_u32(_src26ttt_r.val[1]);
  1725. _dst7.val[0] = vreinterpret_u8_u32(_src37ttt_r.val[1]);
  1726. _dst0.val[1] = vreinterpret_u8_u32(_src04ttt_g.val[0]);
  1727. _dst1.val[1] = vreinterpret_u8_u32(_src15ttt_g.val[0]);
  1728. _dst2.val[1] = vreinterpret_u8_u32(_src26ttt_g.val[0]);
  1729. _dst3.val[1] = vreinterpret_u8_u32(_src37ttt_g.val[0]);
  1730. _dst4.val[1] = vreinterpret_u8_u32(_src04ttt_g.val[1]);
  1731. _dst5.val[1] = vreinterpret_u8_u32(_src15ttt_g.val[1]);
  1732. _dst6.val[1] = vreinterpret_u8_u32(_src26ttt_g.val[1]);
  1733. _dst7.val[1] = vreinterpret_u8_u32(_src37ttt_g.val[1]);
  1734. vst2_u8(dst0, _dst0);
  1735. vst2_u8(dst1, _dst1);
  1736. vst2_u8(dst0 + dst_step, _dst2);
  1737. vst2_u8(dst1 + dst_step, _dst3);
  1738. vst2_u8(dst0 + 2 * dst_step, _dst4);
  1739. vst2_u8(dst1 + 2 * dst_step, _dst5);
  1740. vst2_u8(dst0 + 3 * dst_step, _dst6);
  1741. vst2_u8(dst1 + 3 * dst_step, _dst7);
  1742. src0 += 2 * 8;
  1743. src1 += 2 * 8;
  1744. dst0 += 4 * dst_step;
  1745. dst1 += 4 * dst_step;
  1746. }
  1747. #else
  1748. if (nn > 0)
  1749. {
  1750. asm volatile(
  1751. "0: \n"
  1752. "pld [%1, #128] \n"
  1753. "vld2.u8 {d0-d1}, [%1], %10 \n"
  1754. "pld [%2, #128] \n"
  1755. "vld2.u8 {d2-d3}, [%2], %10 \n"
  1756. "pld [%1, #128] \n"
  1757. "vld2.u8 {d4-d5}, [%1], %10 \n"
  1758. "vtrn.u8 q0, q1 \n" // _src01t_r
  1759. "pld [%2, #128] \n"
  1760. "vld2.u8 {d6-d7}, [%2], %10 \n"
  1761. "pld [%1, #128] \n"
  1762. "vld2.u8 {d16-d17}, [%1], %10\n"
  1763. "vtrn.u8 q2, q3 \n" // _src23t_r
  1764. "pld [%2, #128] \n"
  1765. "vld2.u8 {d18-d19}, [%2], %10\n"
  1766. "pld [%1, #128] \n"
  1767. "vld2.u8 {d20-d21}, [%1], %10\n"
  1768. "vtrn.u8 q8, q9 \n" // _src45t_r
  1769. "pld [%2, #128] \n"
  1770. "vld2.u8 {d22-d23}, [%2], %10\n"
  1771. "vtrn.u8 q10, q11 \n" // _src67t_r
  1772. "sub %1, %1, %10, lsl #2 \n" // restore src0
  1773. "vtrn.u16 q0, q2 \n" // _src02tt_r
  1774. "sub %2, %2, %10, lsl #2 \n" // restore src1
  1775. "vtrn.u16 q1, q3 \n" // _src13tt_r
  1776. "add %1, #16 \n" // src0 += 16
  1777. "vtrn.u16 q8, q10 \n" // _src46tt_r
  1778. "add %2, #16 \n" // src1 += 16
  1779. "vtrn.u16 q9, q11 \n" // _src57tt_r
  1780. "vtrn.u32 q0, q8 \n" // _src04ttt_r
  1781. "vtrn.u32 q1, q9 \n" // _src15ttt_r
  1782. "vst2.u8 {d0-d1}, [%3], %11 \n"
  1783. "vtrn.u32 q2, q10 \n" // _src26ttt_r
  1784. "vst2.u8 {d2-d3}, [%4], %11 \n"
  1785. "vtrn.u32 q3, q11 \n" // _src37ttt_r
  1786. "vst2.u8 {d4-d5}, [%3], %11 \n"
  1787. "subs %0, #1 \n"
  1788. "vst2.u8 {d6-d7}, [%4], %11 \n"
  1789. "vst2.u8 {d16-d17}, [%3], %11\n"
  1790. "vst2.u8 {d18-d19}, [%4], %11\n"
  1791. "vst2.u8 {d20-d21}, [%3], %11\n"
  1792. "vst2.u8 {d22-d23}, [%4], %11\n"
  1793. "bne 0b \n"
  1794. : "=r"(nn), // %0
  1795. "=r"(src0), // %1
  1796. "=r"(src1), // %2
  1797. "=r"(dst0), // %3
  1798. "=r"(dst1) // %4
  1799. : "0"(nn),
  1800. "1"(src0),
  1801. "2"(src1),
  1802. "3"(dst0),
  1803. "4"(dst1),
  1804. "r"(src_step), // %10
  1805. "r"(dst_step) // %11
  1806. : "cc", "memory", "q0", "q1", "q2", "q3", "q8", "q9", "q10", "q11");
  1807. }
  1808. #endif // __aarch64__
  1809. for (; remain > 0; remain--)
  1810. {
  1811. dst0[0] = src0[0];
  1812. dst0[1] = src0[1];
  1813. dst0[2] = src1[0];
  1814. dst0[3] = src1[1];
  1815. dst0[4] = src0[0 + src_step];
  1816. dst0[5] = src0[1 + src_step];
  1817. dst0[6] = src1[0 + src_step];
  1818. dst0[7] = src1[1 + src_step];
  1819. dst0[8] = src0[0 + 2 * src_step];
  1820. dst0[9] = src0[1 + 2 * src_step];
  1821. dst0[10] = src1[0 + 2 * src_step];
  1822. dst0[11] = src1[1 + 2 * src_step];
  1823. dst0[12] = src0[0 + 3 * src_step];
  1824. dst0[13] = src0[1 + 3 * src_step];
  1825. dst0[14] = src1[0 + 3 * src_step];
  1826. dst0[15] = src1[1 + 3 * src_step];
  1827. src0 += 2;
  1828. src1 += 2;
  1829. dst0 += stride;
  1830. }
  1831. src0 += srcwgap + 7 * srcstride;
  1832. }
  1833. #endif // __ARM_NEON
  1834. for (; y < srch; y++)
  1835. {
  1836. unsigned char* dst0 = dst + y * 2;
  1837. int x = 0;
  1838. for (; x < srcw; x++)
  1839. {
  1840. dst0[0] = src0[0];
  1841. dst0[1] = src0[1];
  1842. src0 += 2;
  1843. dst0 += stride;
  1844. }
  1845. src0 += srcwgap;
  1846. }
  1847. }
  1848. static void kanna_rotate_5_c3(const unsigned char* src, int srcw, int srch, int srcstride, unsigned char* dst, int /*w*/, int /*h*/, int stride)
  1849. {
  1850. const int srcwgap = srcstride - srcw * 3;
  1851. const unsigned char* src0 = src;
  1852. int y = 0;
  1853. #if __ARM_NEON
  1854. for (; y + 7 < srch; y += 8)
  1855. {
  1856. const unsigned char* src1 = src0 + srcstride;
  1857. unsigned char* dst0 = dst + y * 3;
  1858. unsigned char* dst1 = dst + y * 3 + stride;
  1859. int src_step = 2 * srcstride;
  1860. int dst_step = 2 * stride;
  1861. int nn = srcw >> 3;
  1862. int remain = srcw - (nn << 3);
  1863. #if !NCNN_GNU_INLINE_ASM || __aarch64__
  1864. for (; nn > 0; nn--)
  1865. {
  1866. uint8x8x3_t _src0 = vld3_u8(src0);
  1867. uint8x8x3_t _src1 = vld3_u8(src1);
  1868. uint8x8x3_t _src2 = vld3_u8(src0 + src_step);
  1869. uint8x8x3_t _src3 = vld3_u8(src1 + src_step);
  1870. uint8x8x3_t _src4 = vld3_u8(src0 + 2 * src_step);
  1871. uint8x8x3_t _src5 = vld3_u8(src1 + 2 * src_step);
  1872. uint8x8x3_t _src6 = vld3_u8(src0 + 3 * src_step);
  1873. uint8x8x3_t _src7 = vld3_u8(src1 + 3 * src_step);
  1874. uint8x8x2_t _src01t_r = vtrn_u8(_src0.val[0], _src1.val[0]);
  1875. uint8x8x2_t _src23t_r = vtrn_u8(_src2.val[0], _src3.val[0]);
  1876. uint8x8x2_t _src45t_r = vtrn_u8(_src4.val[0], _src5.val[0]);
  1877. uint8x8x2_t _src67t_r = vtrn_u8(_src6.val[0], _src7.val[0]);
  1878. uint8x8x2_t _src01t_g = vtrn_u8(_src0.val[1], _src1.val[1]);
  1879. uint8x8x2_t _src23t_g = vtrn_u8(_src2.val[1], _src3.val[1]);
  1880. uint8x8x2_t _src45t_g = vtrn_u8(_src4.val[1], _src5.val[1]);
  1881. uint8x8x2_t _src67t_g = vtrn_u8(_src6.val[1], _src7.val[1]);
  1882. uint8x8x2_t _src01t_b = vtrn_u8(_src0.val[2], _src1.val[2]);
  1883. uint8x8x2_t _src23t_b = vtrn_u8(_src2.val[2], _src3.val[2]);
  1884. uint8x8x2_t _src45t_b = vtrn_u8(_src4.val[2], _src5.val[2]);
  1885. uint8x8x2_t _src67t_b = vtrn_u8(_src6.val[2], _src7.val[2]);
  1886. uint16x4x2_t _src02tt_r = vtrn_u16(vreinterpret_u16_u8(_src01t_r.val[0]), vreinterpret_u16_u8(_src23t_r.val[0]));
  1887. uint16x4x2_t _src13tt_r = vtrn_u16(vreinterpret_u16_u8(_src01t_r.val[1]), vreinterpret_u16_u8(_src23t_r.val[1]));
  1888. uint16x4x2_t _src46tt_r = vtrn_u16(vreinterpret_u16_u8(_src45t_r.val[0]), vreinterpret_u16_u8(_src67t_r.val[0]));
  1889. uint16x4x2_t _src57tt_r = vtrn_u16(vreinterpret_u16_u8(_src45t_r.val[1]), vreinterpret_u16_u8(_src67t_r.val[1]));
  1890. uint16x4x2_t _src02tt_g = vtrn_u16(vreinterpret_u16_u8(_src01t_g.val[0]), vreinterpret_u16_u8(_src23t_g.val[0]));
  1891. uint16x4x2_t _src13tt_g = vtrn_u16(vreinterpret_u16_u8(_src01t_g.val[1]), vreinterpret_u16_u8(_src23t_g.val[1]));
  1892. uint16x4x2_t _src46tt_g = vtrn_u16(vreinterpret_u16_u8(_src45t_g.val[0]), vreinterpret_u16_u8(_src67t_g.val[0]));
  1893. uint16x4x2_t _src57tt_g = vtrn_u16(vreinterpret_u16_u8(_src45t_g.val[1]), vreinterpret_u16_u8(_src67t_g.val[1]));
  1894. uint16x4x2_t _src02tt_b = vtrn_u16(vreinterpret_u16_u8(_src01t_b.val[0]), vreinterpret_u16_u8(_src23t_b.val[0]));
  1895. uint16x4x2_t _src13tt_b = vtrn_u16(vreinterpret_u16_u8(_src01t_b.val[1]), vreinterpret_u16_u8(_src23t_b.val[1]));
  1896. uint16x4x2_t _src46tt_b = vtrn_u16(vreinterpret_u16_u8(_src45t_b.val[0]), vreinterpret_u16_u8(_src67t_b.val[0]));
  1897. uint16x4x2_t _src57tt_b = vtrn_u16(vreinterpret_u16_u8(_src45t_b.val[1]), vreinterpret_u16_u8(_src67t_b.val[1]));
  1898. uint32x2x2_t _src04ttt_r = vtrn_u32(vreinterpret_u32_u16(_src02tt_r.val[0]), vreinterpret_u32_u16(_src46tt_r.val[0]));
  1899. uint32x2x2_t _src15ttt_r = vtrn_u32(vreinterpret_u32_u16(_src13tt_r.val[0]), vreinterpret_u32_u16(_src57tt_r.val[0]));
  1900. uint32x2x2_t _src26ttt_r = vtrn_u32(vreinterpret_u32_u16(_src02tt_r.val[1]), vreinterpret_u32_u16(_src46tt_r.val[1]));
  1901. uint32x2x2_t _src37ttt_r = vtrn_u32(vreinterpret_u32_u16(_src13tt_r.val[1]), vreinterpret_u32_u16(_src57tt_r.val[1]));
  1902. uint32x2x2_t _src04ttt_g = vtrn_u32(vreinterpret_u32_u16(_src02tt_g.val[0]), vreinterpret_u32_u16(_src46tt_g.val[0]));
  1903. uint32x2x2_t _src15ttt_g = vtrn_u32(vreinterpret_u32_u16(_src13tt_g.val[0]), vreinterpret_u32_u16(_src57tt_g.val[0]));
  1904. uint32x2x2_t _src26ttt_g = vtrn_u32(vreinterpret_u32_u16(_src02tt_g.val[1]), vreinterpret_u32_u16(_src46tt_g.val[1]));
  1905. uint32x2x2_t _src37ttt_g = vtrn_u32(vreinterpret_u32_u16(_src13tt_g.val[1]), vreinterpret_u32_u16(_src57tt_g.val[1]));
  1906. uint32x2x2_t _src04ttt_b = vtrn_u32(vreinterpret_u32_u16(_src02tt_b.val[0]), vreinterpret_u32_u16(_src46tt_b.val[0]));
  1907. uint32x2x2_t _src15ttt_b = vtrn_u32(vreinterpret_u32_u16(_src13tt_b.val[0]), vreinterpret_u32_u16(_src57tt_b.val[0]));
  1908. uint32x2x2_t _src26ttt_b = vtrn_u32(vreinterpret_u32_u16(_src02tt_b.val[1]), vreinterpret_u32_u16(_src46tt_b.val[1]));
  1909. uint32x2x2_t _src37ttt_b = vtrn_u32(vreinterpret_u32_u16(_src13tt_b.val[1]), vreinterpret_u32_u16(_src57tt_b.val[1]));
  1910. uint8x8x3_t _dst0;
  1911. uint8x8x3_t _dst1;
  1912. uint8x8x3_t _dst2;
  1913. uint8x8x3_t _dst3;
  1914. uint8x8x3_t _dst4;
  1915. uint8x8x3_t _dst5;
  1916. uint8x8x3_t _dst6;
  1917. uint8x8x3_t _dst7;
  1918. _dst0.val[0] = vreinterpret_u8_u32(_src04ttt_r.val[0]);
  1919. _dst1.val[0] = vreinterpret_u8_u32(_src15ttt_r.val[0]);
  1920. _dst2.val[0] = vreinterpret_u8_u32(_src26ttt_r.val[0]);
  1921. _dst3.val[0] = vreinterpret_u8_u32(_src37ttt_r.val[0]);
  1922. _dst4.val[0] = vreinterpret_u8_u32(_src04ttt_r.val[1]);
  1923. _dst5.val[0] = vreinterpret_u8_u32(_src15ttt_r.val[1]);
  1924. _dst6.val[0] = vreinterpret_u8_u32(_src26ttt_r.val[1]);
  1925. _dst7.val[0] = vreinterpret_u8_u32(_src37ttt_r.val[1]);
  1926. _dst0.val[1] = vreinterpret_u8_u32(_src04ttt_g.val[0]);
  1927. _dst1.val[1] = vreinterpret_u8_u32(_src15ttt_g.val[0]);
  1928. _dst2.val[1] = vreinterpret_u8_u32(_src26ttt_g.val[0]);
  1929. _dst3.val[1] = vreinterpret_u8_u32(_src37ttt_g.val[0]);
  1930. _dst4.val[1] = vreinterpret_u8_u32(_src04ttt_g.val[1]);
  1931. _dst5.val[1] = vreinterpret_u8_u32(_src15ttt_g.val[1]);
  1932. _dst6.val[1] = vreinterpret_u8_u32(_src26ttt_g.val[1]);
  1933. _dst7.val[1] = vreinterpret_u8_u32(_src37ttt_g.val[1]);
  1934. _dst0.val[2] = vreinterpret_u8_u32(_src04ttt_b.val[0]);
  1935. _dst1.val[2] = vreinterpret_u8_u32(_src15ttt_b.val[0]);
  1936. _dst2.val[2] = vreinterpret_u8_u32(_src26ttt_b.val[0]);
  1937. _dst3.val[2] = vreinterpret_u8_u32(_src37ttt_b.val[0]);
  1938. _dst4.val[2] = vreinterpret_u8_u32(_src04ttt_b.val[1]);
  1939. _dst5.val[2] = vreinterpret_u8_u32(_src15ttt_b.val[1]);
  1940. _dst6.val[2] = vreinterpret_u8_u32(_src26ttt_b.val[1]);
  1941. _dst7.val[2] = vreinterpret_u8_u32(_src37ttt_b.val[1]);
  1942. vst3_u8(dst0, _dst0);
  1943. vst3_u8(dst1, _dst1);
  1944. vst3_u8(dst0 + dst_step, _dst2);
  1945. vst3_u8(dst1 + dst_step, _dst3);
  1946. vst3_u8(dst0 + 2 * dst_step, _dst4);
  1947. vst3_u8(dst1 + 2 * dst_step, _dst5);
  1948. vst3_u8(dst0 + 3 * dst_step, _dst6);
  1949. vst3_u8(dst1 + 3 * dst_step, _dst7);
  1950. src0 += 3 * 8;
  1951. src1 += 3 * 8;
  1952. dst0 += 4 * dst_step;
  1953. dst1 += 4 * dst_step;
  1954. }
  1955. #else
  1956. if (nn > 0)
  1957. {
  1958. asm volatile(
  1959. "0: \n"
  1960. "pld [%1, #192] \n"
  1961. "vld3.u8 {d0-d2}, [%1], %10 \n"
  1962. "pld [%2, #192] \n"
  1963. "vld3.u8 {d4-d6}, [%2], %10 \n"
  1964. "pld [%1, #192] \n"
  1965. "vld3.u8 {d8-d10}, [%1], %10 \n"
  1966. "vtrn.u8 q0, q2 \n" // _src01t_r
  1967. "vtrn.u8 d2, d6 \n"
  1968. "pld [%2, #192] \n"
  1969. "vld3.u8 {d12-d14}, [%2], %10\n"
  1970. "pld [%1, #192] \n"
  1971. "vld3.u8 {d16-d18}, [%1], %10\n"
  1972. "vtrn.u8 q4, q6 \n" // _src23t_r
  1973. "vtrn.u8 d10, d14 \n"
  1974. "pld [%2, #192] \n"
  1975. "vld3.u8 {d20-d22}, [%2], %10\n"
  1976. "pld [%1, #192] \n"
  1977. "vld3.u8 {d24-d26}, [%1], %10\n"
  1978. "vtrn.u8 q8, q10 \n" // _src45t_r
  1979. "vtrn.u8 d18, d22 \n"
  1980. "pld [%2, #192] \n"
  1981. "vld3.u8 {d28-d30}, [%2], %10\n"
  1982. "vtrn.u8 q12, q14 \n" // _src67t_r
  1983. "vtrn.u8 d26, d30 \n"
  1984. "sub %1, %1, %10, lsl #2 \n" // restore src0
  1985. "vtrn.u16 q0, q4 \n" // _src02tt_r
  1986. "vtrn.u16 d2, d10 \n"
  1987. "sub %2, %2, %10, lsl #2 \n" // restore src1
  1988. "vtrn.u16 q2, q6 \n" // _src13tt_r
  1989. "vtrn.u16 d6, d14 \n"
  1990. "add %1, #24 \n" // src0 += 24
  1991. "vtrn.u16 q8, q12 \n" // _src46tt_r
  1992. "vtrn.u16 d18, d26 \n"
  1993. "add %2, #24 \n" // src1 += 24
  1994. "vtrn.u16 q10, q14 \n" // _src57tt_r
  1995. "vtrn.u16 d22, d30 \n"
  1996. "vtrn.u32 q0, q8 \n" // _src04ttt_r
  1997. "vtrn.u32 d2, d18 \n"
  1998. "vtrn.u32 q2, q10 \n" // _src15ttt_r
  1999. "vst3.u8 {d0-d2}, [%3], %11 \n"
  2000. "vtrn.u32 d6, d22 \n"
  2001. "vtrn.u32 q4, q12 \n" // _src26ttt_r
  2002. "vst3.u8 {d4-d6}, [%4], %11 \n"
  2003. "vtrn.u32 d10, d26 \n"
  2004. "vtrn.u32 q6, q14 \n" // _src37ttt_r
  2005. "vst3.u8 {d8-d10}, [%3], %11 \n"
  2006. "vtrn.u32 d14, d30 \n"
  2007. "subs %0, #1 \n"
  2008. "vst3.u8 {d16-d18}, [%3], %11\n"
  2009. "vst3.u8 {d12-d14}, [%4], %11\n"
  2010. "vst3.u8 {d20-d22}, [%4], %11\n"
  2011. "vst3.u8 {d24-d26}, [%3], %11\n"
  2012. "vst3.u8 {d28-d30}, [%4], %11\n"
  2013. "bne 0b \n"
  2014. : "=r"(nn), // %0
  2015. "=r"(src0), // %1
  2016. "=r"(src1), // %2
  2017. "=r"(dst0), // %3
  2018. "=r"(dst1) // %4
  2019. : "0"(nn),
  2020. "1"(src0),
  2021. "2"(src1),
  2022. "3"(dst0),
  2023. "4"(dst1),
  2024. "r"(src_step), // %10
  2025. "r"(dst_step) // %11
  2026. : "cc", "memory", "q0", "q1", "q2", "q3", "q4", "q5", "q6", "q7", "q8", "q9", "q10", "q11", "q12", "q13", "q14", "q15");
  2027. }
  2028. #endif // __aarch64__
  2029. for (; remain > 0; remain--)
  2030. {
  2031. dst0[0] = src0[0];
  2032. dst0[1] = src0[1];
  2033. dst0[2] = src0[2];
  2034. dst0[3] = src1[0];
  2035. dst0[4] = src1[1];
  2036. dst0[5] = src1[2];
  2037. dst0[6] = src0[0 + src_step];
  2038. dst0[7] = src0[1 + src_step];
  2039. dst0[8] = src0[2 + src_step];
  2040. dst0[9] = src1[0 + src_step];
  2041. dst0[10] = src1[1 + src_step];
  2042. dst0[11] = src1[2 + src_step];
  2043. dst0[12] = src0[0 + 2 * src_step];
  2044. dst0[13] = src0[1 + 2 * src_step];
  2045. dst0[14] = src0[2 + 2 * src_step];
  2046. dst0[15] = src1[0 + 2 * src_step];
  2047. dst0[16] = src1[1 + 2 * src_step];
  2048. dst0[17] = src1[2 + 2 * src_step];
  2049. dst0[18] = src0[0 + 3 * src_step];
  2050. dst0[19] = src0[1 + 3 * src_step];
  2051. dst0[20] = src0[2 + 3 * src_step];
  2052. dst0[21] = src1[0 + 3 * src_step];
  2053. dst0[22] = src1[1 + 3 * src_step];
  2054. dst0[23] = src1[2 + 3 * src_step];
  2055. src0 += 3;
  2056. src1 += 3;
  2057. dst0 += stride;
  2058. }
  2059. src0 += srcwgap + 7 * srcstride;
  2060. }
  2061. #endif // __ARM_NEON
  2062. for (; y < srch; y++)
  2063. {
  2064. unsigned char* dst0 = dst + y * 3;
  2065. int x = 0;
  2066. for (; x < srcw; x++)
  2067. {
  2068. dst0[0] = src0[0];
  2069. dst0[1] = src0[1];
  2070. dst0[2] = src0[2];
  2071. src0 += 3;
  2072. dst0 += stride;
  2073. }
  2074. src0 += srcwgap;
  2075. }
  2076. }
  2077. static void kanna_rotate_5_c4(const unsigned char* src, int srcw, int srch, int srcstride, unsigned char* dst, int /*w*/, int /*h*/, int stride)
  2078. {
  2079. const int srcwgap = srcstride - srcw * 4;
  2080. const unsigned char* src0 = src;
  2081. int y = 0;
  2082. #if __ARM_NEON
  2083. for (; y + 7 < srch; y += 8)
  2084. {
  2085. const unsigned char* src1 = src0 + srcstride;
  2086. unsigned char* dst0 = dst + y * 4;
  2087. unsigned char* dst1 = dst + y * 4 + stride;
  2088. int src_step = 2 * srcstride;
  2089. int dst_step = 2 * stride;
  2090. int nn = srcw >> 3;
  2091. int remain = srcw - (nn << 3);
  2092. #if !NCNN_GNU_INLINE_ASM || __aarch64__
  2093. for (; nn > 0; nn--)
  2094. {
  2095. uint8x8x4_t _src0 = vld4_u8(src0);
  2096. uint8x8x4_t _src1 = vld4_u8(src1);
  2097. uint8x8x4_t _src2 = vld4_u8(src0 + src_step);
  2098. uint8x8x4_t _src3 = vld4_u8(src1 + src_step);
  2099. uint8x8x4_t _src4 = vld4_u8(src0 + 2 * src_step);
  2100. uint8x8x4_t _src5 = vld4_u8(src1 + 2 * src_step);
  2101. uint8x8x4_t _src6 = vld4_u8(src0 + 3 * src_step);
  2102. uint8x8x4_t _src7 = vld4_u8(src1 + 3 * src_step);
  2103. uint8x8x2_t _src01t_r = vtrn_u8(_src0.val[0], _src1.val[0]);
  2104. uint8x8x2_t _src23t_r = vtrn_u8(_src2.val[0], _src3.val[0]);
  2105. uint8x8x2_t _src45t_r = vtrn_u8(_src4.val[0], _src5.val[0]);
  2106. uint8x8x2_t _src67t_r = vtrn_u8(_src6.val[0], _src7.val[0]);
  2107. uint8x8x2_t _src01t_g = vtrn_u8(_src0.val[1], _src1.val[1]);
  2108. uint8x8x2_t _src23t_g = vtrn_u8(_src2.val[1], _src3.val[1]);
  2109. uint8x8x2_t _src45t_g = vtrn_u8(_src4.val[1], _src5.val[1]);
  2110. uint8x8x2_t _src67t_g = vtrn_u8(_src6.val[1], _src7.val[1]);
  2111. uint8x8x2_t _src01t_b = vtrn_u8(_src0.val[2], _src1.val[2]);
  2112. uint8x8x2_t _src23t_b = vtrn_u8(_src2.val[2], _src3.val[2]);
  2113. uint8x8x2_t _src45t_b = vtrn_u8(_src4.val[2], _src5.val[2]);
  2114. uint8x8x2_t _src67t_b = vtrn_u8(_src6.val[2], _src7.val[2]);
  2115. uint8x8x2_t _src01t_a = vtrn_u8(_src0.val[3], _src1.val[3]);
  2116. uint8x8x2_t _src23t_a = vtrn_u8(_src2.val[3], _src3.val[3]);
  2117. uint8x8x2_t _src45t_a = vtrn_u8(_src4.val[3], _src5.val[3]);
  2118. uint8x8x2_t _src67t_a = vtrn_u8(_src6.val[3], _src7.val[3]);
  2119. uint16x4x2_t _src02tt_r = vtrn_u16(vreinterpret_u16_u8(_src01t_r.val[0]), vreinterpret_u16_u8(_src23t_r.val[0]));
  2120. uint16x4x2_t _src13tt_r = vtrn_u16(vreinterpret_u16_u8(_src01t_r.val[1]), vreinterpret_u16_u8(_src23t_r.val[1]));
  2121. uint16x4x2_t _src46tt_r = vtrn_u16(vreinterpret_u16_u8(_src45t_r.val[0]), vreinterpret_u16_u8(_src67t_r.val[0]));
  2122. uint16x4x2_t _src57tt_r = vtrn_u16(vreinterpret_u16_u8(_src45t_r.val[1]), vreinterpret_u16_u8(_src67t_r.val[1]));
  2123. uint16x4x2_t _src02tt_g = vtrn_u16(vreinterpret_u16_u8(_src01t_g.val[0]), vreinterpret_u16_u8(_src23t_g.val[0]));
  2124. uint16x4x2_t _src13tt_g = vtrn_u16(vreinterpret_u16_u8(_src01t_g.val[1]), vreinterpret_u16_u8(_src23t_g.val[1]));
  2125. uint16x4x2_t _src46tt_g = vtrn_u16(vreinterpret_u16_u8(_src45t_g.val[0]), vreinterpret_u16_u8(_src67t_g.val[0]));
  2126. uint16x4x2_t _src57tt_g = vtrn_u16(vreinterpret_u16_u8(_src45t_g.val[1]), vreinterpret_u16_u8(_src67t_g.val[1]));
  2127. uint16x4x2_t _src02tt_b = vtrn_u16(vreinterpret_u16_u8(_src01t_b.val[0]), vreinterpret_u16_u8(_src23t_b.val[0]));
  2128. uint16x4x2_t _src13tt_b = vtrn_u16(vreinterpret_u16_u8(_src01t_b.val[1]), vreinterpret_u16_u8(_src23t_b.val[1]));
  2129. uint16x4x2_t _src46tt_b = vtrn_u16(vreinterpret_u16_u8(_src45t_b.val[0]), vreinterpret_u16_u8(_src67t_b.val[0]));
  2130. uint16x4x2_t _src57tt_b = vtrn_u16(vreinterpret_u16_u8(_src45t_b.val[1]), vreinterpret_u16_u8(_src67t_b.val[1]));
  2131. uint16x4x2_t _src02tt_a = vtrn_u16(vreinterpret_u16_u8(_src01t_a.val[0]), vreinterpret_u16_u8(_src23t_a.val[0]));
  2132. uint16x4x2_t _src13tt_a = vtrn_u16(vreinterpret_u16_u8(_src01t_a.val[1]), vreinterpret_u16_u8(_src23t_a.val[1]));
  2133. uint16x4x2_t _src46tt_a = vtrn_u16(vreinterpret_u16_u8(_src45t_a.val[0]), vreinterpret_u16_u8(_src67t_a.val[0]));
  2134. uint16x4x2_t _src57tt_a = vtrn_u16(vreinterpret_u16_u8(_src45t_a.val[1]), vreinterpret_u16_u8(_src67t_a.val[1]));
  2135. uint32x2x2_t _src04ttt_r = vtrn_u32(vreinterpret_u32_u16(_src02tt_r.val[0]), vreinterpret_u32_u16(_src46tt_r.val[0]));
  2136. uint32x2x2_t _src15ttt_r = vtrn_u32(vreinterpret_u32_u16(_src13tt_r.val[0]), vreinterpret_u32_u16(_src57tt_r.val[0]));
  2137. uint32x2x2_t _src26ttt_r = vtrn_u32(vreinterpret_u32_u16(_src02tt_r.val[1]), vreinterpret_u32_u16(_src46tt_r.val[1]));
  2138. uint32x2x2_t _src37ttt_r = vtrn_u32(vreinterpret_u32_u16(_src13tt_r.val[1]), vreinterpret_u32_u16(_src57tt_r.val[1]));
  2139. uint32x2x2_t _src04ttt_g = vtrn_u32(vreinterpret_u32_u16(_src02tt_g.val[0]), vreinterpret_u32_u16(_src46tt_g.val[0]));
  2140. uint32x2x2_t _src15ttt_g = vtrn_u32(vreinterpret_u32_u16(_src13tt_g.val[0]), vreinterpret_u32_u16(_src57tt_g.val[0]));
  2141. uint32x2x2_t _src26ttt_g = vtrn_u32(vreinterpret_u32_u16(_src02tt_g.val[1]), vreinterpret_u32_u16(_src46tt_g.val[1]));
  2142. uint32x2x2_t _src37ttt_g = vtrn_u32(vreinterpret_u32_u16(_src13tt_g.val[1]), vreinterpret_u32_u16(_src57tt_g.val[1]));
  2143. uint32x2x2_t _src04ttt_b = vtrn_u32(vreinterpret_u32_u16(_src02tt_b.val[0]), vreinterpret_u32_u16(_src46tt_b.val[0]));
  2144. uint32x2x2_t _src15ttt_b = vtrn_u32(vreinterpret_u32_u16(_src13tt_b.val[0]), vreinterpret_u32_u16(_src57tt_b.val[0]));
  2145. uint32x2x2_t _src26ttt_b = vtrn_u32(vreinterpret_u32_u16(_src02tt_b.val[1]), vreinterpret_u32_u16(_src46tt_b.val[1]));
  2146. uint32x2x2_t _src37ttt_b = vtrn_u32(vreinterpret_u32_u16(_src13tt_b.val[1]), vreinterpret_u32_u16(_src57tt_b.val[1]));
  2147. uint32x2x2_t _src04ttt_a = vtrn_u32(vreinterpret_u32_u16(_src02tt_a.val[0]), vreinterpret_u32_u16(_src46tt_a.val[0]));
  2148. uint32x2x2_t _src15ttt_a = vtrn_u32(vreinterpret_u32_u16(_src13tt_a.val[0]), vreinterpret_u32_u16(_src57tt_a.val[0]));
  2149. uint32x2x2_t _src26ttt_a = vtrn_u32(vreinterpret_u32_u16(_src02tt_a.val[1]), vreinterpret_u32_u16(_src46tt_a.val[1]));
  2150. uint32x2x2_t _src37ttt_a = vtrn_u32(vreinterpret_u32_u16(_src13tt_a.val[1]), vreinterpret_u32_u16(_src57tt_a.val[1]));
  2151. uint8x8x4_t _dst0;
  2152. uint8x8x4_t _dst1;
  2153. uint8x8x4_t _dst2;
  2154. uint8x8x4_t _dst3;
  2155. uint8x8x4_t _dst4;
  2156. uint8x8x4_t _dst5;
  2157. uint8x8x4_t _dst6;
  2158. uint8x8x4_t _dst7;
  2159. _dst0.val[0] = vreinterpret_u8_u32(_src04ttt_r.val[0]);
  2160. _dst1.val[0] = vreinterpret_u8_u32(_src15ttt_r.val[0]);
  2161. _dst2.val[0] = vreinterpret_u8_u32(_src26ttt_r.val[0]);
  2162. _dst3.val[0] = vreinterpret_u8_u32(_src37ttt_r.val[0]);
  2163. _dst4.val[0] = vreinterpret_u8_u32(_src04ttt_r.val[1]);
  2164. _dst5.val[0] = vreinterpret_u8_u32(_src15ttt_r.val[1]);
  2165. _dst6.val[0] = vreinterpret_u8_u32(_src26ttt_r.val[1]);
  2166. _dst7.val[0] = vreinterpret_u8_u32(_src37ttt_r.val[1]);
  2167. _dst0.val[1] = vreinterpret_u8_u32(_src04ttt_g.val[0]);
  2168. _dst1.val[1] = vreinterpret_u8_u32(_src15ttt_g.val[0]);
  2169. _dst2.val[1] = vreinterpret_u8_u32(_src26ttt_g.val[0]);
  2170. _dst3.val[1] = vreinterpret_u8_u32(_src37ttt_g.val[0]);
  2171. _dst4.val[1] = vreinterpret_u8_u32(_src04ttt_g.val[1]);
  2172. _dst5.val[1] = vreinterpret_u8_u32(_src15ttt_g.val[1]);
  2173. _dst6.val[1] = vreinterpret_u8_u32(_src26ttt_g.val[1]);
  2174. _dst7.val[1] = vreinterpret_u8_u32(_src37ttt_g.val[1]);
  2175. _dst0.val[2] = vreinterpret_u8_u32(_src04ttt_b.val[0]);
  2176. _dst1.val[2] = vreinterpret_u8_u32(_src15ttt_b.val[0]);
  2177. _dst2.val[2] = vreinterpret_u8_u32(_src26ttt_b.val[0]);
  2178. _dst3.val[2] = vreinterpret_u8_u32(_src37ttt_b.val[0]);
  2179. _dst4.val[2] = vreinterpret_u8_u32(_src04ttt_b.val[1]);
  2180. _dst5.val[2] = vreinterpret_u8_u32(_src15ttt_b.val[1]);
  2181. _dst6.val[2] = vreinterpret_u8_u32(_src26ttt_b.val[1]);
  2182. _dst7.val[2] = vreinterpret_u8_u32(_src37ttt_b.val[1]);
  2183. _dst0.val[3] = vreinterpret_u8_u32(_src04ttt_a.val[0]);
  2184. _dst1.val[3] = vreinterpret_u8_u32(_src15ttt_a.val[0]);
  2185. _dst2.val[3] = vreinterpret_u8_u32(_src26ttt_a.val[0]);
  2186. _dst3.val[3] = vreinterpret_u8_u32(_src37ttt_a.val[0]);
  2187. _dst4.val[3] = vreinterpret_u8_u32(_src04ttt_a.val[1]);
  2188. _dst5.val[3] = vreinterpret_u8_u32(_src15ttt_a.val[1]);
  2189. _dst6.val[3] = vreinterpret_u8_u32(_src26ttt_a.val[1]);
  2190. _dst7.val[3] = vreinterpret_u8_u32(_src37ttt_a.val[1]);
  2191. vst4_u8(dst0, _dst0);
  2192. vst4_u8(dst1, _dst1);
  2193. vst4_u8(dst0 + dst_step, _dst2);
  2194. vst4_u8(dst1 + dst_step, _dst3);
  2195. vst4_u8(dst0 + 2 * dst_step, _dst4);
  2196. vst4_u8(dst1 + 2 * dst_step, _dst5);
  2197. vst4_u8(dst0 + 3 * dst_step, _dst6);
  2198. vst4_u8(dst1 + 3 * dst_step, _dst7);
  2199. src0 += 4 * 8;
  2200. src1 += 4 * 8;
  2201. dst0 += 4 * dst_step;
  2202. dst1 += 4 * dst_step;
  2203. }
  2204. #else
  2205. if (nn > 0)
  2206. {
  2207. asm volatile(
  2208. "0: \n"
  2209. "pld [%1, #256] \n"
  2210. "vld4.u8 {d0-d3}, [%1], %10 \n"
  2211. "pld [%2, #256] \n"
  2212. "vld4.u8 {d4-d7}, [%2], %10 \n"
  2213. "pld [%1, #256] \n"
  2214. "vld4.u8 {d8-d11}, [%1], %10 \n"
  2215. "vtrn.u8 q0, q2 \n" // _src01t_r
  2216. "vtrn.u8 q1, q3 \n"
  2217. "pld [%2, #256] \n"
  2218. "vld4.u8 {d12-d15}, [%2], %10\n"
  2219. "pld [%1, #256] \n"
  2220. "vld4.u8 {d16-d19}, [%1], %10\n"
  2221. "vtrn.u8 q4, q6 \n" // _src23t_r
  2222. "vtrn.u8 q5, q7 \n"
  2223. "pld [%2, #256] \n"
  2224. "vld4.u8 {d20-d23}, [%2], %10\n"
  2225. "pld [%1, #256] \n"
  2226. "vld4.u8 {d24-d27}, [%1], %10\n"
  2227. "vtrn.u8 q8, q10 \n" // _src45t_r
  2228. "vtrn.u8 q9, q11 \n"
  2229. "pld [%2, #256] \n"
  2230. "vld4.u8 {d28-d31}, [%2], %10\n"
  2231. "vtrn.u8 q12, q14 \n" // _src67t_r
  2232. "vtrn.u8 q13, q15 \n"
  2233. "sub %1, %1, %10, lsl #2 \n" // restore src0
  2234. "vtrn.u16 q0, q4 \n" // _src02tt_r
  2235. "vtrn.u16 q1, q5 \n"
  2236. "sub %2, %2, %10, lsl #2 \n" // restore src1
  2237. "vtrn.u16 q2, q6 \n" // _src13tt_r
  2238. "vtrn.u16 q3, q7 \n"
  2239. "add %1, #32 \n" // src0 += 32
  2240. "vtrn.u16 q8, q12 \n" // _src46tt_r
  2241. "vtrn.u16 q9, q13 \n"
  2242. "add %2, #32 \n" // src1 += 32
  2243. "vtrn.u16 q10, q14 \n" // _src57tt_r
  2244. "vtrn.u16 q11, q15 \n"
  2245. "vtrn.u32 q0, q8 \n" // _src04ttt_r
  2246. "vtrn.u32 q1, q9 \n"
  2247. "vtrn.u32 q2, q10 \n" // _src15ttt_r
  2248. "vst4.u8 {d0-d3}, [%3], %11 \n"
  2249. "vtrn.u32 q3, q11 \n"
  2250. "vtrn.u32 q4, q12 \n" // _src26ttt_r
  2251. "vst4.u8 {d4-d7}, [%4], %11 \n"
  2252. "vtrn.u32 q5, q13 \n"
  2253. "vtrn.u32 q6, q14 \n" // _src37ttt_r
  2254. "vst4.u8 {d8-d11}, [%3], %11 \n"
  2255. "vtrn.u32 q7, q15 \n"
  2256. "subs %0, #1 \n"
  2257. "vst4.u8 {d16-d19}, [%3], %11\n"
  2258. "vst4.u8 {d12-d15}, [%4], %11\n"
  2259. "vst4.u8 {d20-d23}, [%4], %11\n"
  2260. "vst4.u8 {d24-d27}, [%3], %11\n"
  2261. "vst4.u8 {d28-d31}, [%4], %11\n"
  2262. "bne 0b \n"
  2263. : "=r"(nn), // %0
  2264. "=r"(src0), // %1
  2265. "=r"(src1), // %2
  2266. "=r"(dst0), // %3
  2267. "=r"(dst1) // %4
  2268. : "0"(nn),
  2269. "1"(src0),
  2270. "2"(src1),
  2271. "3"(dst0),
  2272. "4"(dst1),
  2273. "r"(src_step), // %10
  2274. "r"(dst_step) // %11
  2275. : "cc", "memory", "q0", "q1", "q2", "q3", "q4", "q5", "q6", "q7", "q8", "q9", "q10", "q11", "q12", "q13", "q14", "q15");
  2276. }
  2277. #endif // __aarch64__
  2278. for (; remain > 0; remain--)
  2279. {
  2280. dst0[0] = src0[0];
  2281. dst0[1] = src0[1];
  2282. dst0[2] = src0[2];
  2283. dst0[3] = src0[3];
  2284. dst0[4] = src1[0];
  2285. dst0[5] = src1[1];
  2286. dst0[6] = src1[2];
  2287. dst0[7] = src1[3];
  2288. dst0[8] = src0[0 + src_step];
  2289. dst0[9] = src0[1 + src_step];
  2290. dst0[10] = src0[2 + src_step];
  2291. dst0[11] = src0[3 + src_step];
  2292. dst0[12] = src1[0 + src_step];
  2293. dst0[13] = src1[1 + src_step];
  2294. dst0[14] = src1[2 + src_step];
  2295. dst0[15] = src1[3 + src_step];
  2296. dst0[16] = src0[0 + 2 * src_step];
  2297. dst0[17] = src0[1 + 2 * src_step];
  2298. dst0[18] = src0[2 + 2 * src_step];
  2299. dst0[19] = src0[3 + 2 * src_step];
  2300. dst0[20] = src1[0 + 2 * src_step];
  2301. dst0[21] = src1[1 + 2 * src_step];
  2302. dst0[22] = src1[2 + 2 * src_step];
  2303. dst0[23] = src1[3 + 2 * src_step];
  2304. dst0[24] = src0[0 + 3 * src_step];
  2305. dst0[25] = src0[1 + 3 * src_step];
  2306. dst0[26] = src0[2 + 3 * src_step];
  2307. dst0[27] = src0[3 + 3 * src_step];
  2308. dst0[28] = src1[0 + 3 * src_step];
  2309. dst0[29] = src1[1 + 3 * src_step];
  2310. dst0[30] = src1[2 + 3 * src_step];
  2311. dst0[31] = src1[3 + 3 * src_step];
  2312. src0 += 4;
  2313. src1 += 4;
  2314. dst0 += stride;
  2315. }
  2316. src0 += srcwgap + 7 * srcstride;
  2317. }
  2318. #endif // __ARM_NEON
  2319. for (; y < srch; y++)
  2320. {
  2321. unsigned char* dst0 = dst + y * 4;
  2322. int x = 0;
  2323. for (; x < srcw; x++)
  2324. {
  2325. dst0[0] = src0[0];
  2326. dst0[1] = src0[1];
  2327. dst0[2] = src0[2];
  2328. dst0[3] = src0[3];
  2329. src0 += 4;
  2330. dst0 += stride;
  2331. }
  2332. src0 += srcwgap;
  2333. }
  2334. }
  2335. static void kanna_rotate_6_c1(const unsigned char* src, int srcw, int srch, int srcstride, unsigned char* dst, int w, int /*h*/, int stride)
  2336. {
  2337. const int srcwgap = srcstride - srcw;
  2338. // point to the last dst pixel in row
  2339. unsigned char* dstend = dst + w;
  2340. const unsigned char* src0 = src;
  2341. int y = 0;
  2342. #if __ARM_NEON
  2343. for (; y + 7 < srch; y += 8)
  2344. {
  2345. const unsigned char* src1 = src0 + srcstride;
  2346. unsigned char* dst0 = dstend - y - 8;
  2347. unsigned char* dst1 = dstend - y - 8 + stride;
  2348. int src_step = 2 * srcstride;
  2349. int dst_step = 2 * stride;
  2350. int nn = srcw >> 3;
  2351. int remain = srcw - (nn << 3);
  2352. #if !NCNN_GNU_INLINE_ASM || __aarch64__
  2353. for (; nn > 0; nn--)
  2354. {
  2355. uint8x8_t _src0 = vld1_u8(src0);
  2356. uint8x8_t _src1 = vld1_u8(src1);
  2357. uint8x8_t _src2 = vld1_u8(src0 + src_step);
  2358. uint8x8_t _src3 = vld1_u8(src1 + src_step);
  2359. uint8x8_t _src4 = vld1_u8(src0 + 2 * src_step);
  2360. uint8x8_t _src5 = vld1_u8(src1 + 2 * src_step);
  2361. uint8x8_t _src6 = vld1_u8(src0 + 3 * src_step);
  2362. uint8x8_t _src7 = vld1_u8(src1 + 3 * src_step);
  2363. uint8x8x2_t _src01t_r = vtrn_u8(_src1, _src0);
  2364. uint8x8x2_t _src23t_r = vtrn_u8(_src3, _src2);
  2365. uint8x8x2_t _src45t_r = vtrn_u8(_src5, _src4);
  2366. uint8x8x2_t _src67t_r = vtrn_u8(_src7, _src6);
  2367. uint16x4x2_t _src02tt_r = vtrn_u16(vreinterpret_u16_u8(_src23t_r.val[1]), vreinterpret_u16_u8(_src01t_r.val[1]));
  2368. uint16x4x2_t _src13tt_r = vtrn_u16(vreinterpret_u16_u8(_src23t_r.val[0]), vreinterpret_u16_u8(_src01t_r.val[0]));
  2369. uint16x4x2_t _src46tt_r = vtrn_u16(vreinterpret_u16_u8(_src67t_r.val[1]), vreinterpret_u16_u8(_src45t_r.val[1]));
  2370. uint16x4x2_t _src57tt_r = vtrn_u16(vreinterpret_u16_u8(_src67t_r.val[0]), vreinterpret_u16_u8(_src45t_r.val[0]));
  2371. uint32x2x2_t _src04ttt_r = vtrn_u32(vreinterpret_u32_u16(_src46tt_r.val[1]), vreinterpret_u32_u16(_src02tt_r.val[1]));
  2372. uint32x2x2_t _src15ttt_r = vtrn_u32(vreinterpret_u32_u16(_src57tt_r.val[1]), vreinterpret_u32_u16(_src13tt_r.val[1]));
  2373. uint32x2x2_t _src26ttt_r = vtrn_u32(vreinterpret_u32_u16(_src46tt_r.val[0]), vreinterpret_u32_u16(_src02tt_r.val[0]));
  2374. uint32x2x2_t _src37ttt_r = vtrn_u32(vreinterpret_u32_u16(_src57tt_r.val[0]), vreinterpret_u32_u16(_src13tt_r.val[0]));
  2375. uint8x8_t _dst0 = vreinterpret_u8_u32(_src04ttt_r.val[1]);
  2376. uint8x8_t _dst1 = vreinterpret_u8_u32(_src15ttt_r.val[1]);
  2377. uint8x8_t _dst2 = vreinterpret_u8_u32(_src26ttt_r.val[1]);
  2378. uint8x8_t _dst3 = vreinterpret_u8_u32(_src37ttt_r.val[1]);
  2379. uint8x8_t _dst4 = vreinterpret_u8_u32(_src04ttt_r.val[0]);
  2380. uint8x8_t _dst5 = vreinterpret_u8_u32(_src15ttt_r.val[0]);
  2381. uint8x8_t _dst6 = vreinterpret_u8_u32(_src26ttt_r.val[0]);
  2382. uint8x8_t _dst7 = vreinterpret_u8_u32(_src37ttt_r.val[0]);
  2383. vst1_u8(dst0, _dst7);
  2384. vst1_u8(dst1, _dst6);
  2385. vst1_u8(dst0 + dst_step, _dst5);
  2386. vst1_u8(dst1 + dst_step, _dst4);
  2387. vst1_u8(dst0 + 2 * dst_step, _dst3);
  2388. vst1_u8(dst1 + 2 * dst_step, _dst2);
  2389. vst1_u8(dst0 + 3 * dst_step, _dst1);
  2390. vst1_u8(dst1 + 3 * dst_step, _dst0);
  2391. src0 += 8;
  2392. src1 += 8;
  2393. dst0 += 4 * dst_step;
  2394. dst1 += 4 * dst_step;
  2395. }
  2396. #else
  2397. if (nn > 0)
  2398. {
  2399. asm volatile(
  2400. "0: \n"
  2401. "pld [%1, #64] \n"
  2402. "vld1.u8 {d0}, [%1], %10 \n"
  2403. "pld [%2, #64] \n"
  2404. "vld1.u8 {d1}, [%2], %10 \n"
  2405. "pld [%1, #64] \n"
  2406. "vld1.u8 {d2}, [%1], %10 \n"
  2407. "vtrn.u8 d1, d0 \n" // _src01t_r
  2408. "pld [%2, #64] \n"
  2409. "vld1.u8 {d3}, [%2], %10 \n"
  2410. "pld [%1, #64] \n"
  2411. "vld1.u8 {d4}, [%1], %10 \n"
  2412. "vtrn.u8 d3, d2 \n" // _src23t_r
  2413. "pld [%2, #64] \n"
  2414. "vld1.u8 {d5}, [%2], %10 \n"
  2415. "pld [%1, #64] \n"
  2416. "vld1.u8 {d6}, [%1], %10 \n"
  2417. "vtrn.u8 d5, d4 \n" // _src45t_r
  2418. "pld [%2, #64] \n"
  2419. "vld1.u8 {d7}, [%2], %10 \n"
  2420. "vtrn.u8 d7, d6 \n" // _src67t_r
  2421. "sub %1, %1, %10, lsl #2 \n" // restore src0
  2422. "vtrn.u16 q1, q0 \n" // _src02tt_r _src13tt_r
  2423. "sub %2, %2, %10, lsl #2 \n" // restore src1
  2424. "vtrn.u16 q3, q2 \n" // _src46tt_r _src57tt_r
  2425. "add %1, #8 \n" // src0 += 8
  2426. "vtrn.u32 q3, q1 \n" // _src26ttt_r _src37ttt_r
  2427. "add %2, #8 \n" // src1 += 8
  2428. "vtrn.u32 q2, q0 \n" // _src04ttt_r _src15ttt_r
  2429. "vst1.u8 {d6}, [%4], %11 \n"
  2430. "vst1.u8 {d7}, [%3], %11 \n"
  2431. "subs %0, #1 \n"
  2432. "vst1.u8 {d4}, [%4], %11 \n"
  2433. "vst1.u8 {d5}, [%3], %11 \n"
  2434. "vst1.u8 {d2}, [%4], %11 \n"
  2435. "vst1.u8 {d3}, [%3], %11 \n"
  2436. "vst1.u8 {d0}, [%4], %11 \n"
  2437. "vst1.u8 {d1}, [%3], %11 \n"
  2438. "bne 0b \n"
  2439. : "=r"(nn), // %0
  2440. "=r"(src0), // %1
  2441. "=r"(src1), // %2
  2442. "=r"(dst0), // %3
  2443. "=r"(dst1) // %4
  2444. : "0"(nn),
  2445. "1"(src0),
  2446. "2"(src1),
  2447. "3"(dst0),
  2448. "4"(dst1),
  2449. "r"(src_step), // %10
  2450. "r"(dst_step) // %11
  2451. : "cc", "memory", "q0", "q1", "q2", "q3");
  2452. }
  2453. #endif // __aarch64__
  2454. for (; remain > 0; remain--)
  2455. {
  2456. dst0[0] = src1[0 + 3 * src_step];
  2457. dst0[1] = src0[0 + 3 * src_step];
  2458. dst0[2] = src1[0 + 2 * src_step];
  2459. dst0[3] = src0[0 + 2 * src_step];
  2460. dst0[4] = src1[0 + src_step];
  2461. dst0[5] = src0[0 + src_step];
  2462. dst0[6] = src1[0];
  2463. dst0[7] = src0[0];
  2464. src0 += 1;
  2465. src1 += 1;
  2466. dst0 += stride;
  2467. }
  2468. src0 += srcwgap + 7 * srcstride;
  2469. }
  2470. #endif // __ARM_NEON
  2471. for (; y < srch; y++)
  2472. {
  2473. unsigned char* dst0 = dstend - y - 1;
  2474. int x = 0;
  2475. for (; x < srcw; x++)
  2476. {
  2477. *dst0 = *src0;
  2478. src0 += 1;
  2479. dst0 += stride;
  2480. }
  2481. src0 += srcwgap;
  2482. }
  2483. }
  2484. static void kanna_rotate_6_c2(const unsigned char* src, int srcw, int srch, int srcstride, unsigned char* dst, int w, int /*h*/, int stride)
  2485. {
  2486. const int srcwgap = srcstride - srcw * 2;
  2487. // point to the last dst pixel in row
  2488. unsigned char* dstend = dst + w * 2;
  2489. const unsigned char* src0 = src;
  2490. int y = 0;
  2491. #if __ARM_NEON
  2492. for (; y + 7 < srch; y += 8)
  2493. {
  2494. const unsigned char* src1 = src0 + srcstride;
  2495. unsigned char* dst0 = dstend - y * 2 - 8 * 2;
  2496. unsigned char* dst1 = dstend - y * 2 - 8 * 2 + stride;
  2497. int src_step = 2 * srcstride;
  2498. int dst_step = 2 * stride;
  2499. int nn = srcw >> 3;
  2500. int remain = srcw - (nn << 3);
  2501. #if !NCNN_GNU_INLINE_ASM || __aarch64__
  2502. for (; nn > 0; nn--)
  2503. {
  2504. uint8x8x2_t _src0 = vld2_u8(src0);
  2505. uint8x8x2_t _src1 = vld2_u8(src1);
  2506. uint8x8x2_t _src2 = vld2_u8(src0 + src_step);
  2507. uint8x8x2_t _src3 = vld2_u8(src1 + src_step);
  2508. uint8x8x2_t _src4 = vld2_u8(src0 + 2 * src_step);
  2509. uint8x8x2_t _src5 = vld2_u8(src1 + 2 * src_step);
  2510. uint8x8x2_t _src6 = vld2_u8(src0 + 3 * src_step);
  2511. uint8x8x2_t _src7 = vld2_u8(src1 + 3 * src_step);
  2512. uint8x8x2_t _src01t_r = vtrn_u8(_src1.val[0], _src0.val[0]);
  2513. uint8x8x2_t _src23t_r = vtrn_u8(_src3.val[0], _src2.val[0]);
  2514. uint8x8x2_t _src45t_r = vtrn_u8(_src5.val[0], _src4.val[0]);
  2515. uint8x8x2_t _src67t_r = vtrn_u8(_src7.val[0], _src6.val[0]);
  2516. uint8x8x2_t _src01t_g = vtrn_u8(_src1.val[1], _src0.val[1]);
  2517. uint8x8x2_t _src23t_g = vtrn_u8(_src3.val[1], _src2.val[1]);
  2518. uint8x8x2_t _src45t_g = vtrn_u8(_src5.val[1], _src4.val[1]);
  2519. uint8x8x2_t _src67t_g = vtrn_u8(_src7.val[1], _src6.val[1]);
  2520. uint16x4x2_t _src02tt_r = vtrn_u16(vreinterpret_u16_u8(_src23t_r.val[1]), vreinterpret_u16_u8(_src01t_r.val[1]));
  2521. uint16x4x2_t _src13tt_r = vtrn_u16(vreinterpret_u16_u8(_src23t_r.val[0]), vreinterpret_u16_u8(_src01t_r.val[0]));
  2522. uint16x4x2_t _src46tt_r = vtrn_u16(vreinterpret_u16_u8(_src67t_r.val[1]), vreinterpret_u16_u8(_src45t_r.val[1]));
  2523. uint16x4x2_t _src57tt_r = vtrn_u16(vreinterpret_u16_u8(_src67t_r.val[0]), vreinterpret_u16_u8(_src45t_r.val[0]));
  2524. uint16x4x2_t _src02tt_g = vtrn_u16(vreinterpret_u16_u8(_src23t_g.val[1]), vreinterpret_u16_u8(_src01t_g.val[1]));
  2525. uint16x4x2_t _src13tt_g = vtrn_u16(vreinterpret_u16_u8(_src23t_g.val[0]), vreinterpret_u16_u8(_src01t_g.val[0]));
  2526. uint16x4x2_t _src46tt_g = vtrn_u16(vreinterpret_u16_u8(_src67t_g.val[1]), vreinterpret_u16_u8(_src45t_g.val[1]));
  2527. uint16x4x2_t _src57tt_g = vtrn_u16(vreinterpret_u16_u8(_src67t_g.val[0]), vreinterpret_u16_u8(_src45t_g.val[0]));
  2528. uint32x2x2_t _src04ttt_r = vtrn_u32(vreinterpret_u32_u16(_src46tt_r.val[1]), vreinterpret_u32_u16(_src02tt_r.val[1]));
  2529. uint32x2x2_t _src15ttt_r = vtrn_u32(vreinterpret_u32_u16(_src57tt_r.val[1]), vreinterpret_u32_u16(_src13tt_r.val[1]));
  2530. uint32x2x2_t _src26ttt_r = vtrn_u32(vreinterpret_u32_u16(_src46tt_r.val[0]), vreinterpret_u32_u16(_src02tt_r.val[0]));
  2531. uint32x2x2_t _src37ttt_r = vtrn_u32(vreinterpret_u32_u16(_src57tt_r.val[0]), vreinterpret_u32_u16(_src13tt_r.val[0]));
  2532. uint32x2x2_t _src04ttt_g = vtrn_u32(vreinterpret_u32_u16(_src46tt_g.val[1]), vreinterpret_u32_u16(_src02tt_g.val[1]));
  2533. uint32x2x2_t _src15ttt_g = vtrn_u32(vreinterpret_u32_u16(_src57tt_g.val[1]), vreinterpret_u32_u16(_src13tt_g.val[1]));
  2534. uint32x2x2_t _src26ttt_g = vtrn_u32(vreinterpret_u32_u16(_src46tt_g.val[0]), vreinterpret_u32_u16(_src02tt_g.val[0]));
  2535. uint32x2x2_t _src37ttt_g = vtrn_u32(vreinterpret_u32_u16(_src57tt_g.val[0]), vreinterpret_u32_u16(_src13tt_g.val[0]));
  2536. uint8x8x2_t _dst0;
  2537. uint8x8x2_t _dst1;
  2538. uint8x8x2_t _dst2;
  2539. uint8x8x2_t _dst3;
  2540. uint8x8x2_t _dst4;
  2541. uint8x8x2_t _dst5;
  2542. uint8x8x2_t _dst6;
  2543. uint8x8x2_t _dst7;
  2544. _dst0.val[0] = vreinterpret_u8_u32(_src04ttt_r.val[1]);
  2545. _dst1.val[0] = vreinterpret_u8_u32(_src15ttt_r.val[1]);
  2546. _dst2.val[0] = vreinterpret_u8_u32(_src26ttt_r.val[1]);
  2547. _dst3.val[0] = vreinterpret_u8_u32(_src37ttt_r.val[1]);
  2548. _dst4.val[0] = vreinterpret_u8_u32(_src04ttt_r.val[0]);
  2549. _dst5.val[0] = vreinterpret_u8_u32(_src15ttt_r.val[0]);
  2550. _dst6.val[0] = vreinterpret_u8_u32(_src26ttt_r.val[0]);
  2551. _dst7.val[0] = vreinterpret_u8_u32(_src37ttt_r.val[0]);
  2552. _dst0.val[1] = vreinterpret_u8_u32(_src04ttt_g.val[1]);
  2553. _dst1.val[1] = vreinterpret_u8_u32(_src15ttt_g.val[1]);
  2554. _dst2.val[1] = vreinterpret_u8_u32(_src26ttt_g.val[1]);
  2555. _dst3.val[1] = vreinterpret_u8_u32(_src37ttt_g.val[1]);
  2556. _dst4.val[1] = vreinterpret_u8_u32(_src04ttt_g.val[0]);
  2557. _dst5.val[1] = vreinterpret_u8_u32(_src15ttt_g.val[0]);
  2558. _dst6.val[1] = vreinterpret_u8_u32(_src26ttt_g.val[0]);
  2559. _dst7.val[1] = vreinterpret_u8_u32(_src37ttt_g.val[0]);
  2560. vst2_u8(dst0, _dst7);
  2561. vst2_u8(dst1, _dst6);
  2562. vst2_u8(dst0 + dst_step, _dst5);
  2563. vst2_u8(dst1 + dst_step, _dst4);
  2564. vst2_u8(dst0 + 2 * dst_step, _dst3);
  2565. vst2_u8(dst1 + 2 * dst_step, _dst2);
  2566. vst2_u8(dst0 + 3 * dst_step, _dst1);
  2567. vst2_u8(dst1 + 3 * dst_step, _dst0);
  2568. src0 += 2 * 8;
  2569. src1 += 2 * 8;
  2570. dst0 += 4 * dst_step;
  2571. dst1 += 4 * dst_step;
  2572. }
  2573. #else
  2574. if (nn > 0)
  2575. {
  2576. asm volatile(
  2577. "0: \n"
  2578. "pld [%1, #128] \n"
  2579. "vld2.u8 {d0-d1}, [%1], %10 \n"
  2580. "pld [%2, #128] \n"
  2581. "vld2.u8 {d2-d3}, [%2], %10 \n"
  2582. "pld [%1, #128] \n"
  2583. "vld2.u8 {d4-d5}, [%1], %10 \n"
  2584. "vtrn.u8 q1, q0 \n" // _src01t_r
  2585. "pld [%2, #128] \n"
  2586. "vld2.u8 {d6-d7}, [%2], %10 \n"
  2587. "pld [%1, #128] \n"
  2588. "vld2.u8 {d16-d17}, [%1], %10\n"
  2589. "vtrn.u8 q3, q2 \n" // _src23t_r
  2590. "pld [%2, #128] \n"
  2591. "vld2.u8 {d18-d19}, [%2], %10\n"
  2592. "pld [%1, #128] \n"
  2593. "vld2.u8 {d20-d21}, [%1], %10\n"
  2594. "vtrn.u8 q9, q8 \n" // _src45t_r
  2595. "pld [%2, #128] \n"
  2596. "vld2.u8 {d22-d23}, [%2], %10\n"
  2597. "vtrn.u8 q11, q10 \n" // _src67t_r
  2598. "sub %1, %1, %10, lsl #2 \n" // restore src0
  2599. "vtrn.u16 q2, q0 \n" // _src02tt_r
  2600. "sub %2, %2, %10, lsl #2 \n" // restore src1
  2601. "vtrn.u16 q3, q1 \n" // _src13tt_r
  2602. "add %1, #16 \n" // src0 += 16
  2603. "vtrn.u16 q10, q8 \n" // _src46tt_r
  2604. "add %2, #16 \n" // src1 += 16
  2605. "vtrn.u16 q11, q9 \n" // _src57tt_r
  2606. "vtrn.u32 q10, q2 \n" // _src26ttt_r
  2607. "vtrn.u32 q11, q3 \n" // _src37ttt_r
  2608. "vst2.u8 {d20-d21}, [%4], %11\n"
  2609. "vtrn.u32 q8, q0 \n" // _src04ttt_r
  2610. "vst2.u8 {d22-d23}, [%3], %11\n"
  2611. "vtrn.u32 q9, q1 \n" // _src15ttt_r
  2612. "vst2.u8 {d16-d17}, [%4], %11\n"
  2613. "subs %0, #1 \n"
  2614. "vst2.u8 {d18-d19}, [%3], %11\n"
  2615. "vst2.u8 {d4-d5}, [%4], %11 \n"
  2616. "vst2.u8 {d6-d7}, [%3], %11 \n"
  2617. "vst2.u8 {d0-d1}, [%4], %11 \n"
  2618. "vst2.u8 {d2-d3}, [%3], %11 \n"
  2619. "bne 0b \n"
  2620. : "=r"(nn), // %0
  2621. "=r"(src0), // %1
  2622. "=r"(src1), // %2
  2623. "=r"(dst0), // %3
  2624. "=r"(dst1) // %4
  2625. : "0"(nn),
  2626. "1"(src0),
  2627. "2"(src1),
  2628. "3"(dst0),
  2629. "4"(dst1),
  2630. "r"(src_step), // %10
  2631. "r"(dst_step) // %11
  2632. : "cc", "memory", "q0", "q1", "q2", "q3", "q8", "q9", "q10", "q11");
  2633. }
  2634. #endif // __aarch64__
  2635. for (; remain > 0; remain--)
  2636. {
  2637. dst0[0] = src1[0 + 3 * src_step];
  2638. dst0[1] = src1[1 + 3 * src_step];
  2639. dst0[2] = src0[0 + 3 * src_step];
  2640. dst0[3] = src0[1 + 3 * src_step];
  2641. dst0[4] = src1[0 + 2 * src_step];
  2642. dst0[5] = src1[1 + 2 * src_step];
  2643. dst0[6] = src0[0 + 2 * src_step];
  2644. dst0[7] = src0[1 + 2 * src_step];
  2645. dst0[8] = src1[0 + src_step];
  2646. dst0[9] = src1[1 + src_step];
  2647. dst0[10] = src0[0 + src_step];
  2648. dst0[11] = src0[1 + src_step];
  2649. dst0[12] = src1[0];
  2650. dst0[13] = src1[1];
  2651. dst0[14] = src0[0];
  2652. dst0[15] = src0[1];
  2653. src0 += 2;
  2654. src1 += 2;
  2655. dst0 += stride;
  2656. }
  2657. src0 += srcwgap + 7 * srcstride;
  2658. }
  2659. #endif // __ARM_NEON
  2660. for (; y < srch; y++)
  2661. {
  2662. unsigned char* dst0 = dstend - y * 2 - 2;
  2663. int x = 0;
  2664. for (; x < srcw; x++)
  2665. {
  2666. dst0[0] = src0[0];
  2667. dst0[1] = src0[1];
  2668. src0 += 2;
  2669. dst0 += stride;
  2670. }
  2671. src0 += srcwgap;
  2672. }
  2673. }
  2674. static void kanna_rotate_6_c3(const unsigned char* src, int srcw, int srch, int srcstride, unsigned char* dst, int w, int /*h*/, int stride)
  2675. {
  2676. const int srcwgap = srcstride - srcw * 3;
  2677. // point to the last dst pixel in row
  2678. unsigned char* dstend = dst + w * 3;
  2679. const unsigned char* src0 = src;
  2680. int y = 0;
  2681. #if __ARM_NEON
  2682. for (; y + 7 < srch; y += 8)
  2683. {
  2684. const unsigned char* src1 = src0 + srcstride;
  2685. unsigned char* dst0 = dstend - y * 3 - 8 * 3;
  2686. unsigned char* dst1 = dstend - y * 3 - 8 * 3 + stride;
  2687. int src_step = 2 * srcstride;
  2688. int dst_step = 2 * stride;
  2689. int nn = srcw >> 3;
  2690. int remain = srcw - (nn << 3);
  2691. #if !NCNN_GNU_INLINE_ASM || __aarch64__
  2692. for (; nn > 0; nn--)
  2693. {
  2694. uint8x8x3_t _src0 = vld3_u8(src0);
  2695. uint8x8x3_t _src1 = vld3_u8(src1);
  2696. uint8x8x3_t _src2 = vld3_u8(src0 + src_step);
  2697. uint8x8x3_t _src3 = vld3_u8(src1 + src_step);
  2698. uint8x8x3_t _src4 = vld3_u8(src0 + 2 * src_step);
  2699. uint8x8x3_t _src5 = vld3_u8(src1 + 2 * src_step);
  2700. uint8x8x3_t _src6 = vld3_u8(src0 + 3 * src_step);
  2701. uint8x8x3_t _src7 = vld3_u8(src1 + 3 * src_step);
  2702. uint8x8x2_t _src01t_r = vtrn_u8(_src1.val[0], _src0.val[0]);
  2703. uint8x8x2_t _src23t_r = vtrn_u8(_src3.val[0], _src2.val[0]);
  2704. uint8x8x2_t _src45t_r = vtrn_u8(_src5.val[0], _src4.val[0]);
  2705. uint8x8x2_t _src67t_r = vtrn_u8(_src7.val[0], _src6.val[0]);
  2706. uint8x8x2_t _src01t_g = vtrn_u8(_src1.val[1], _src0.val[1]);
  2707. uint8x8x2_t _src23t_g = vtrn_u8(_src3.val[1], _src2.val[1]);
  2708. uint8x8x2_t _src45t_g = vtrn_u8(_src5.val[1], _src4.val[1]);
  2709. uint8x8x2_t _src67t_g = vtrn_u8(_src7.val[1], _src6.val[1]);
  2710. uint8x8x2_t _src01t_b = vtrn_u8(_src1.val[2], _src0.val[2]);
  2711. uint8x8x2_t _src23t_b = vtrn_u8(_src3.val[2], _src2.val[2]);
  2712. uint8x8x2_t _src45t_b = vtrn_u8(_src5.val[2], _src4.val[2]);
  2713. uint8x8x2_t _src67t_b = vtrn_u8(_src7.val[2], _src6.val[2]);
  2714. uint16x4x2_t _src02tt_r = vtrn_u16(vreinterpret_u16_u8(_src23t_r.val[1]), vreinterpret_u16_u8(_src01t_r.val[1]));
  2715. uint16x4x2_t _src13tt_r = vtrn_u16(vreinterpret_u16_u8(_src23t_r.val[0]), vreinterpret_u16_u8(_src01t_r.val[0]));
  2716. uint16x4x2_t _src46tt_r = vtrn_u16(vreinterpret_u16_u8(_src67t_r.val[1]), vreinterpret_u16_u8(_src45t_r.val[1]));
  2717. uint16x4x2_t _src57tt_r = vtrn_u16(vreinterpret_u16_u8(_src67t_r.val[0]), vreinterpret_u16_u8(_src45t_r.val[0]));
  2718. uint16x4x2_t _src02tt_g = vtrn_u16(vreinterpret_u16_u8(_src23t_g.val[1]), vreinterpret_u16_u8(_src01t_g.val[1]));
  2719. uint16x4x2_t _src13tt_g = vtrn_u16(vreinterpret_u16_u8(_src23t_g.val[0]), vreinterpret_u16_u8(_src01t_g.val[0]));
  2720. uint16x4x2_t _src46tt_g = vtrn_u16(vreinterpret_u16_u8(_src67t_g.val[1]), vreinterpret_u16_u8(_src45t_g.val[1]));
  2721. uint16x4x2_t _src57tt_g = vtrn_u16(vreinterpret_u16_u8(_src67t_g.val[0]), vreinterpret_u16_u8(_src45t_g.val[0]));
  2722. uint16x4x2_t _src02tt_b = vtrn_u16(vreinterpret_u16_u8(_src23t_b.val[1]), vreinterpret_u16_u8(_src01t_b.val[1]));
  2723. uint16x4x2_t _src13tt_b = vtrn_u16(vreinterpret_u16_u8(_src23t_b.val[0]), vreinterpret_u16_u8(_src01t_b.val[0]));
  2724. uint16x4x2_t _src46tt_b = vtrn_u16(vreinterpret_u16_u8(_src67t_b.val[1]), vreinterpret_u16_u8(_src45t_b.val[1]));
  2725. uint16x4x2_t _src57tt_b = vtrn_u16(vreinterpret_u16_u8(_src67t_b.val[0]), vreinterpret_u16_u8(_src45t_b.val[0]));
  2726. uint32x2x2_t _src04ttt_r = vtrn_u32(vreinterpret_u32_u16(_src46tt_r.val[1]), vreinterpret_u32_u16(_src02tt_r.val[1]));
  2727. uint32x2x2_t _src15ttt_r = vtrn_u32(vreinterpret_u32_u16(_src57tt_r.val[1]), vreinterpret_u32_u16(_src13tt_r.val[1]));
  2728. uint32x2x2_t _src26ttt_r = vtrn_u32(vreinterpret_u32_u16(_src46tt_r.val[0]), vreinterpret_u32_u16(_src02tt_r.val[0]));
  2729. uint32x2x2_t _src37ttt_r = vtrn_u32(vreinterpret_u32_u16(_src57tt_r.val[0]), vreinterpret_u32_u16(_src13tt_r.val[0]));
  2730. uint32x2x2_t _src04ttt_g = vtrn_u32(vreinterpret_u32_u16(_src46tt_g.val[1]), vreinterpret_u32_u16(_src02tt_g.val[1]));
  2731. uint32x2x2_t _src15ttt_g = vtrn_u32(vreinterpret_u32_u16(_src57tt_g.val[1]), vreinterpret_u32_u16(_src13tt_g.val[1]));
  2732. uint32x2x2_t _src26ttt_g = vtrn_u32(vreinterpret_u32_u16(_src46tt_g.val[0]), vreinterpret_u32_u16(_src02tt_g.val[0]));
  2733. uint32x2x2_t _src37ttt_g = vtrn_u32(vreinterpret_u32_u16(_src57tt_g.val[0]), vreinterpret_u32_u16(_src13tt_g.val[0]));
  2734. uint32x2x2_t _src04ttt_b = vtrn_u32(vreinterpret_u32_u16(_src46tt_b.val[1]), vreinterpret_u32_u16(_src02tt_b.val[1]));
  2735. uint32x2x2_t _src15ttt_b = vtrn_u32(vreinterpret_u32_u16(_src57tt_b.val[1]), vreinterpret_u32_u16(_src13tt_b.val[1]));
  2736. uint32x2x2_t _src26ttt_b = vtrn_u32(vreinterpret_u32_u16(_src46tt_b.val[0]), vreinterpret_u32_u16(_src02tt_b.val[0]));
  2737. uint32x2x2_t _src37ttt_b = vtrn_u32(vreinterpret_u32_u16(_src57tt_b.val[0]), vreinterpret_u32_u16(_src13tt_b.val[0]));
  2738. uint8x8x3_t _dst0;
  2739. uint8x8x3_t _dst1;
  2740. uint8x8x3_t _dst2;
  2741. uint8x8x3_t _dst3;
  2742. uint8x8x3_t _dst4;
  2743. uint8x8x3_t _dst5;
  2744. uint8x8x3_t _dst6;
  2745. uint8x8x3_t _dst7;
  2746. _dst0.val[0] = vreinterpret_u8_u32(_src04ttt_r.val[1]);
  2747. _dst1.val[0] = vreinterpret_u8_u32(_src15ttt_r.val[1]);
  2748. _dst2.val[0] = vreinterpret_u8_u32(_src26ttt_r.val[1]);
  2749. _dst3.val[0] = vreinterpret_u8_u32(_src37ttt_r.val[1]);
  2750. _dst4.val[0] = vreinterpret_u8_u32(_src04ttt_r.val[0]);
  2751. _dst5.val[0] = vreinterpret_u8_u32(_src15ttt_r.val[0]);
  2752. _dst6.val[0] = vreinterpret_u8_u32(_src26ttt_r.val[0]);
  2753. _dst7.val[0] = vreinterpret_u8_u32(_src37ttt_r.val[0]);
  2754. _dst0.val[1] = vreinterpret_u8_u32(_src04ttt_g.val[1]);
  2755. _dst1.val[1] = vreinterpret_u8_u32(_src15ttt_g.val[1]);
  2756. _dst2.val[1] = vreinterpret_u8_u32(_src26ttt_g.val[1]);
  2757. _dst3.val[1] = vreinterpret_u8_u32(_src37ttt_g.val[1]);
  2758. _dst4.val[1] = vreinterpret_u8_u32(_src04ttt_g.val[0]);
  2759. _dst5.val[1] = vreinterpret_u8_u32(_src15ttt_g.val[0]);
  2760. _dst6.val[1] = vreinterpret_u8_u32(_src26ttt_g.val[0]);
  2761. _dst7.val[1] = vreinterpret_u8_u32(_src37ttt_g.val[0]);
  2762. _dst0.val[2] = vreinterpret_u8_u32(_src04ttt_b.val[1]);
  2763. _dst1.val[2] = vreinterpret_u8_u32(_src15ttt_b.val[1]);
  2764. _dst2.val[2] = vreinterpret_u8_u32(_src26ttt_b.val[1]);
  2765. _dst3.val[2] = vreinterpret_u8_u32(_src37ttt_b.val[1]);
  2766. _dst4.val[2] = vreinterpret_u8_u32(_src04ttt_b.val[0]);
  2767. _dst5.val[2] = vreinterpret_u8_u32(_src15ttt_b.val[0]);
  2768. _dst6.val[2] = vreinterpret_u8_u32(_src26ttt_b.val[0]);
  2769. _dst7.val[2] = vreinterpret_u8_u32(_src37ttt_b.val[0]);
  2770. vst3_u8(dst0, _dst7);
  2771. vst3_u8(dst1, _dst6);
  2772. vst3_u8(dst0 + dst_step, _dst5);
  2773. vst3_u8(dst1 + dst_step, _dst4);
  2774. vst3_u8(dst0 + 2 * dst_step, _dst3);
  2775. vst3_u8(dst1 + 2 * dst_step, _dst2);
  2776. vst3_u8(dst0 + 3 * dst_step, _dst1);
  2777. vst3_u8(dst1 + 3 * dst_step, _dst0);
  2778. src0 += 3 * 8;
  2779. src1 += 3 * 8;
  2780. dst0 += 4 * dst_step;
  2781. dst1 += 4 * dst_step;
  2782. }
  2783. #else
  2784. if (nn > 0)
  2785. {
  2786. asm volatile(
  2787. "0: \n"
  2788. "pld [%1, #192] \n"
  2789. "vld3.u8 {d0-d2}, [%1], %10 \n"
  2790. "pld [%2, #192] \n"
  2791. "vld3.u8 {d4-d6}, [%2], %10 \n"
  2792. "pld [%1, #192] \n"
  2793. "vld3.u8 {d8-d10}, [%1], %10 \n"
  2794. "vtrn.u8 q2, q0 \n" // _src01t_r
  2795. "vtrn.u8 d6, d2 \n"
  2796. "pld [%2, #192] \n"
  2797. "vld3.u8 {d12-d14}, [%2], %10\n"
  2798. "pld [%1, #192] \n"
  2799. "vld3.u8 {d16-d18}, [%1], %10\n"
  2800. "vtrn.u8 q6, q4 \n" // _src23t_r
  2801. "vtrn.u8 d14, d10 \n"
  2802. "pld [%2, #192] \n"
  2803. "vld3.u8 {d20-d22}, [%2], %10\n"
  2804. "pld [%1, #192] \n"
  2805. "vld3.u8 {d24-d26}, [%1], %10\n"
  2806. "vtrn.u8 q10, q8 \n" // _src45t_r
  2807. "vtrn.u8 d22, d18 \n"
  2808. "pld [%2, #192] \n"
  2809. "vld3.u8 {d28-d30}, [%2], %10\n"
  2810. "vtrn.u8 q14, q12 \n" // _src67t_r
  2811. "vtrn.u8 d30, d26 \n"
  2812. "sub %1, %1, %10, lsl #2 \n" // restore src0
  2813. "vtrn.u16 q4, q0 \n" // _src02tt_r
  2814. "vtrn.u16 d10, d2 \n"
  2815. "sub %2, %2, %10, lsl #2 \n" // restore src1
  2816. "vtrn.u16 q6, q2 \n" // _src13tt_r
  2817. "vtrn.u16 d14, d6 \n"
  2818. "add %1, #24 \n" // src0 += 24
  2819. "vtrn.u16 q12, q8 \n" // _src46tt_r
  2820. "vtrn.u16 d26, d18 \n"
  2821. "add %2, #24 \n" // src1 += 24
  2822. "vtrn.u16 q14, q10 \n" // _src57tt_r
  2823. "vtrn.u16 d30, d22 \n"
  2824. "vtrn.u32 q12, q4 \n" // _src26ttt_r
  2825. "vtrn.u32 d26, d10 \n"
  2826. "vtrn.u32 q14, q6 \n" // _src37ttt_r
  2827. "vst3.u8 {d24-d26}, [%4], %11\n"
  2828. "vtrn.u32 d30, d14 \n"
  2829. "vtrn.u32 q8, q0 \n" // _src04ttt_r
  2830. "vst3.u8 {d28-d30}, [%3], %11\n"
  2831. "vtrn.u32 d18, d2 \n"
  2832. "vtrn.u32 q10, q2 \n" // _src15ttt_r
  2833. "vst3.u8 {d16-d18}, [%4], %11\n"
  2834. "vtrn.u32 d22, d6 \n"
  2835. "subs %0, #1 \n"
  2836. "vst3.u8 {d20-d22}, [%3], %11\n"
  2837. "vst3.u8 {d8-d10}, [%4], %11 \n"
  2838. "vst3.u8 {d12-d14}, [%3], %11\n"
  2839. "vst3.u8 {d0-d2}, [%4], %11 \n"
  2840. "vst3.u8 {d4-d6}, [%3], %11 \n"
  2841. "bne 0b \n"
  2842. : "=r"(nn), // %0
  2843. "=r"(src0), // %1
  2844. "=r"(src1), // %2
  2845. "=r"(dst0), // %3
  2846. "=r"(dst1) // %4
  2847. : "0"(nn),
  2848. "1"(src0),
  2849. "2"(src1),
  2850. "3"(dst0),
  2851. "4"(dst1),
  2852. "r"(src_step), // %10
  2853. "r"(dst_step) // %11
  2854. : "cc", "memory", "q0", "q1", "q2", "q3", "q4", "q5", "q6", "q7", "q8", "q9", "q10", "q11", "q12", "q13", "q14", "q15");
  2855. }
  2856. #endif // __aarch64__
  2857. for (; remain > 0; remain--)
  2858. {
  2859. dst0[0] = src1[0 + 3 * src_step];
  2860. dst0[1] = src1[1 + 3 * src_step];
  2861. dst0[2] = src1[2 + 3 * src_step];
  2862. dst0[3] = src0[0 + 3 * src_step];
  2863. dst0[4] = src0[1 + 3 * src_step];
  2864. dst0[5] = src0[2 + 3 * src_step];
  2865. dst0[6] = src1[0 + 2 * src_step];
  2866. dst0[7] = src1[1 + 2 * src_step];
  2867. dst0[8] = src1[2 + 2 * src_step];
  2868. dst0[9] = src0[0 + 2 * src_step];
  2869. dst0[10] = src0[1 + 2 * src_step];
  2870. dst0[11] = src0[2 + 2 * src_step];
  2871. dst0[12] = src1[0 + src_step];
  2872. dst0[13] = src1[1 + src_step];
  2873. dst0[14] = src1[2 + src_step];
  2874. dst0[15] = src0[0 + src_step];
  2875. dst0[16] = src0[1 + src_step];
  2876. dst0[17] = src0[2 + src_step];
  2877. dst0[18] = src1[0];
  2878. dst0[19] = src1[1];
  2879. dst0[20] = src1[2];
  2880. dst0[21] = src0[0];
  2881. dst0[22] = src0[1];
  2882. dst0[23] = src0[2];
  2883. src0 += 3;
  2884. src1 += 3;
  2885. dst0 += stride;
  2886. }
  2887. src0 += srcwgap + 7 * srcstride;
  2888. }
  2889. #endif // __ARM_NEON
  2890. for (; y < srch; y++)
  2891. {
  2892. unsigned char* dst0 = dstend - y * 3 - 3;
  2893. int x = 0;
  2894. for (; x < srcw; x++)
  2895. {
  2896. dst0[0] = src0[0];
  2897. dst0[1] = src0[1];
  2898. dst0[2] = src0[2];
  2899. src0 += 3;
  2900. dst0 += stride;
  2901. }
  2902. src0 += srcwgap;
  2903. }
  2904. }
  2905. static void kanna_rotate_6_c4(const unsigned char* src, int srcw, int srch, int srcstride, unsigned char* dst, int w, int /*h*/, int stride)
  2906. {
  2907. const int srcwgap = srcstride - srcw * 4;
  2908. // point to the last dst pixel in row
  2909. unsigned char* dstend = dst + w * 4;
  2910. const unsigned char* src0 = src;
  2911. int y = 0;
  2912. #if __ARM_NEON
  2913. for (; y + 7 < srch; y += 8)
  2914. {
  2915. const unsigned char* src1 = src0 + srcstride;
  2916. unsigned char* dst0 = dstend - y * 4 - 8 * 4;
  2917. unsigned char* dst1 = dstend - y * 4 - 8 * 4 + stride;
  2918. int src_step = 2 * srcstride;
  2919. int dst_step = 2 * stride;
  2920. int nn = srcw >> 3;
  2921. int remain = srcw - (nn << 3);
  2922. #if !NCNN_GNU_INLINE_ASM || __aarch64__
  2923. for (; nn > 0; nn--)
  2924. {
  2925. uint8x8x4_t _src0 = vld4_u8(src0);
  2926. uint8x8x4_t _src1 = vld4_u8(src1);
  2927. uint8x8x4_t _src2 = vld4_u8(src0 + src_step);
  2928. uint8x8x4_t _src3 = vld4_u8(src1 + src_step);
  2929. uint8x8x4_t _src4 = vld4_u8(src0 + 2 * src_step);
  2930. uint8x8x4_t _src5 = vld4_u8(src1 + 2 * src_step);
  2931. uint8x8x4_t _src6 = vld4_u8(src0 + 3 * src_step);
  2932. uint8x8x4_t _src7 = vld4_u8(src1 + 3 * src_step);
  2933. uint8x8x2_t _src01t_r = vtrn_u8(_src1.val[0], _src0.val[0]);
  2934. uint8x8x2_t _src23t_r = vtrn_u8(_src3.val[0], _src2.val[0]);
  2935. uint8x8x2_t _src45t_r = vtrn_u8(_src5.val[0], _src4.val[0]);
  2936. uint8x8x2_t _src67t_r = vtrn_u8(_src7.val[0], _src6.val[0]);
  2937. uint8x8x2_t _src01t_g = vtrn_u8(_src1.val[1], _src0.val[1]);
  2938. uint8x8x2_t _src23t_g = vtrn_u8(_src3.val[1], _src2.val[1]);
  2939. uint8x8x2_t _src45t_g = vtrn_u8(_src5.val[1], _src4.val[1]);
  2940. uint8x8x2_t _src67t_g = vtrn_u8(_src7.val[1], _src6.val[1]);
  2941. uint8x8x2_t _src01t_b = vtrn_u8(_src1.val[2], _src0.val[2]);
  2942. uint8x8x2_t _src23t_b = vtrn_u8(_src3.val[2], _src2.val[2]);
  2943. uint8x8x2_t _src45t_b = vtrn_u8(_src5.val[2], _src4.val[2]);
  2944. uint8x8x2_t _src67t_b = vtrn_u8(_src7.val[2], _src6.val[2]);
  2945. uint8x8x2_t _src01t_a = vtrn_u8(_src1.val[3], _src0.val[3]);
  2946. uint8x8x2_t _src23t_a = vtrn_u8(_src3.val[3], _src2.val[3]);
  2947. uint8x8x2_t _src45t_a = vtrn_u8(_src5.val[3], _src4.val[3]);
  2948. uint8x8x2_t _src67t_a = vtrn_u8(_src7.val[3], _src6.val[3]);
  2949. uint16x4x2_t _src02tt_r = vtrn_u16(vreinterpret_u16_u8(_src23t_r.val[1]), vreinterpret_u16_u8(_src01t_r.val[1]));
  2950. uint16x4x2_t _src13tt_r = vtrn_u16(vreinterpret_u16_u8(_src23t_r.val[0]), vreinterpret_u16_u8(_src01t_r.val[0]));
  2951. uint16x4x2_t _src46tt_r = vtrn_u16(vreinterpret_u16_u8(_src67t_r.val[1]), vreinterpret_u16_u8(_src45t_r.val[1]));
  2952. uint16x4x2_t _src57tt_r = vtrn_u16(vreinterpret_u16_u8(_src67t_r.val[0]), vreinterpret_u16_u8(_src45t_r.val[0]));
  2953. uint16x4x2_t _src02tt_g = vtrn_u16(vreinterpret_u16_u8(_src23t_g.val[1]), vreinterpret_u16_u8(_src01t_g.val[1]));
  2954. uint16x4x2_t _src13tt_g = vtrn_u16(vreinterpret_u16_u8(_src23t_g.val[0]), vreinterpret_u16_u8(_src01t_g.val[0]));
  2955. uint16x4x2_t _src46tt_g = vtrn_u16(vreinterpret_u16_u8(_src67t_g.val[1]), vreinterpret_u16_u8(_src45t_g.val[1]));
  2956. uint16x4x2_t _src57tt_g = vtrn_u16(vreinterpret_u16_u8(_src67t_g.val[0]), vreinterpret_u16_u8(_src45t_g.val[0]));
  2957. uint16x4x2_t _src02tt_b = vtrn_u16(vreinterpret_u16_u8(_src23t_b.val[1]), vreinterpret_u16_u8(_src01t_b.val[1]));
  2958. uint16x4x2_t _src13tt_b = vtrn_u16(vreinterpret_u16_u8(_src23t_b.val[0]), vreinterpret_u16_u8(_src01t_b.val[0]));
  2959. uint16x4x2_t _src46tt_b = vtrn_u16(vreinterpret_u16_u8(_src67t_b.val[1]), vreinterpret_u16_u8(_src45t_b.val[1]));
  2960. uint16x4x2_t _src57tt_b = vtrn_u16(vreinterpret_u16_u8(_src67t_b.val[0]), vreinterpret_u16_u8(_src45t_b.val[0]));
  2961. uint16x4x2_t _src02tt_a = vtrn_u16(vreinterpret_u16_u8(_src23t_a.val[1]), vreinterpret_u16_u8(_src01t_a.val[1]));
  2962. uint16x4x2_t _src13tt_a = vtrn_u16(vreinterpret_u16_u8(_src23t_a.val[0]), vreinterpret_u16_u8(_src01t_a.val[0]));
  2963. uint16x4x2_t _src46tt_a = vtrn_u16(vreinterpret_u16_u8(_src67t_a.val[1]), vreinterpret_u16_u8(_src45t_a.val[1]));
  2964. uint16x4x2_t _src57tt_a = vtrn_u16(vreinterpret_u16_u8(_src67t_a.val[0]), vreinterpret_u16_u8(_src45t_a.val[0]));
  2965. uint32x2x2_t _src04ttt_r = vtrn_u32(vreinterpret_u32_u16(_src46tt_r.val[1]), vreinterpret_u32_u16(_src02tt_r.val[1]));
  2966. uint32x2x2_t _src15ttt_r = vtrn_u32(vreinterpret_u32_u16(_src57tt_r.val[1]), vreinterpret_u32_u16(_src13tt_r.val[1]));
  2967. uint32x2x2_t _src26ttt_r = vtrn_u32(vreinterpret_u32_u16(_src46tt_r.val[0]), vreinterpret_u32_u16(_src02tt_r.val[0]));
  2968. uint32x2x2_t _src37ttt_r = vtrn_u32(vreinterpret_u32_u16(_src57tt_r.val[0]), vreinterpret_u32_u16(_src13tt_r.val[0]));
  2969. uint32x2x2_t _src04ttt_g = vtrn_u32(vreinterpret_u32_u16(_src46tt_g.val[1]), vreinterpret_u32_u16(_src02tt_g.val[1]));
  2970. uint32x2x2_t _src15ttt_g = vtrn_u32(vreinterpret_u32_u16(_src57tt_g.val[1]), vreinterpret_u32_u16(_src13tt_g.val[1]));
  2971. uint32x2x2_t _src26ttt_g = vtrn_u32(vreinterpret_u32_u16(_src46tt_g.val[0]), vreinterpret_u32_u16(_src02tt_g.val[0]));
  2972. uint32x2x2_t _src37ttt_g = vtrn_u32(vreinterpret_u32_u16(_src57tt_g.val[0]), vreinterpret_u32_u16(_src13tt_g.val[0]));
  2973. uint32x2x2_t _src04ttt_b = vtrn_u32(vreinterpret_u32_u16(_src46tt_b.val[1]), vreinterpret_u32_u16(_src02tt_b.val[1]));
  2974. uint32x2x2_t _src15ttt_b = vtrn_u32(vreinterpret_u32_u16(_src57tt_b.val[1]), vreinterpret_u32_u16(_src13tt_b.val[1]));
  2975. uint32x2x2_t _src26ttt_b = vtrn_u32(vreinterpret_u32_u16(_src46tt_b.val[0]), vreinterpret_u32_u16(_src02tt_b.val[0]));
  2976. uint32x2x2_t _src37ttt_b = vtrn_u32(vreinterpret_u32_u16(_src57tt_b.val[0]), vreinterpret_u32_u16(_src13tt_b.val[0]));
  2977. uint32x2x2_t _src04ttt_a = vtrn_u32(vreinterpret_u32_u16(_src46tt_a.val[1]), vreinterpret_u32_u16(_src02tt_a.val[1]));
  2978. uint32x2x2_t _src15ttt_a = vtrn_u32(vreinterpret_u32_u16(_src57tt_a.val[1]), vreinterpret_u32_u16(_src13tt_a.val[1]));
  2979. uint32x2x2_t _src26ttt_a = vtrn_u32(vreinterpret_u32_u16(_src46tt_a.val[0]), vreinterpret_u32_u16(_src02tt_a.val[0]));
  2980. uint32x2x2_t _src37ttt_a = vtrn_u32(vreinterpret_u32_u16(_src57tt_a.val[0]), vreinterpret_u32_u16(_src13tt_a.val[0]));
  2981. uint8x8x4_t _dst0;
  2982. uint8x8x4_t _dst1;
  2983. uint8x8x4_t _dst2;
  2984. uint8x8x4_t _dst3;
  2985. uint8x8x4_t _dst4;
  2986. uint8x8x4_t _dst5;
  2987. uint8x8x4_t _dst6;
  2988. uint8x8x4_t _dst7;
  2989. _dst0.val[0] = vreinterpret_u8_u32(_src04ttt_r.val[1]);
  2990. _dst1.val[0] = vreinterpret_u8_u32(_src15ttt_r.val[1]);
  2991. _dst2.val[0] = vreinterpret_u8_u32(_src26ttt_r.val[1]);
  2992. _dst3.val[0] = vreinterpret_u8_u32(_src37ttt_r.val[1]);
  2993. _dst4.val[0] = vreinterpret_u8_u32(_src04ttt_r.val[0]);
  2994. _dst5.val[0] = vreinterpret_u8_u32(_src15ttt_r.val[0]);
  2995. _dst6.val[0] = vreinterpret_u8_u32(_src26ttt_r.val[0]);
  2996. _dst7.val[0] = vreinterpret_u8_u32(_src37ttt_r.val[0]);
  2997. _dst0.val[1] = vreinterpret_u8_u32(_src04ttt_g.val[1]);
  2998. _dst1.val[1] = vreinterpret_u8_u32(_src15ttt_g.val[1]);
  2999. _dst2.val[1] = vreinterpret_u8_u32(_src26ttt_g.val[1]);
  3000. _dst3.val[1] = vreinterpret_u8_u32(_src37ttt_g.val[1]);
  3001. _dst4.val[1] = vreinterpret_u8_u32(_src04ttt_g.val[0]);
  3002. _dst5.val[1] = vreinterpret_u8_u32(_src15ttt_g.val[0]);
  3003. _dst6.val[1] = vreinterpret_u8_u32(_src26ttt_g.val[0]);
  3004. _dst7.val[1] = vreinterpret_u8_u32(_src37ttt_g.val[0]);
  3005. _dst0.val[2] = vreinterpret_u8_u32(_src04ttt_b.val[1]);
  3006. _dst1.val[2] = vreinterpret_u8_u32(_src15ttt_b.val[1]);
  3007. _dst2.val[2] = vreinterpret_u8_u32(_src26ttt_b.val[1]);
  3008. _dst3.val[2] = vreinterpret_u8_u32(_src37ttt_b.val[1]);
  3009. _dst4.val[2] = vreinterpret_u8_u32(_src04ttt_b.val[0]);
  3010. _dst5.val[2] = vreinterpret_u8_u32(_src15ttt_b.val[0]);
  3011. _dst6.val[2] = vreinterpret_u8_u32(_src26ttt_b.val[0]);
  3012. _dst7.val[2] = vreinterpret_u8_u32(_src37ttt_b.val[0]);
  3013. _dst0.val[3] = vreinterpret_u8_u32(_src04ttt_a.val[1]);
  3014. _dst1.val[3] = vreinterpret_u8_u32(_src15ttt_a.val[1]);
  3015. _dst2.val[3] = vreinterpret_u8_u32(_src26ttt_a.val[1]);
  3016. _dst3.val[3] = vreinterpret_u8_u32(_src37ttt_a.val[1]);
  3017. _dst4.val[3] = vreinterpret_u8_u32(_src04ttt_a.val[0]);
  3018. _dst5.val[3] = vreinterpret_u8_u32(_src15ttt_a.val[0]);
  3019. _dst6.val[3] = vreinterpret_u8_u32(_src26ttt_a.val[0]);
  3020. _dst7.val[3] = vreinterpret_u8_u32(_src37ttt_a.val[0]);
  3021. vst4_u8(dst0, _dst7);
  3022. vst4_u8(dst1, _dst6);
  3023. vst4_u8(dst0 + dst_step, _dst5);
  3024. vst4_u8(dst1 + dst_step, _dst4);
  3025. vst4_u8(dst0 + 2 * dst_step, _dst3);
  3026. vst4_u8(dst1 + 2 * dst_step, _dst2);
  3027. vst4_u8(dst0 + 3 * dst_step, _dst1);
  3028. vst4_u8(dst1 + 3 * dst_step, _dst0);
  3029. src0 += 4 * 8;
  3030. src1 += 4 * 8;
  3031. dst0 += 4 * dst_step;
  3032. dst1 += 4 * dst_step;
  3033. }
  3034. #else
  3035. if (nn > 0)
  3036. {
  3037. asm volatile(
  3038. "0: \n"
  3039. "pld [%1, #256] \n"
  3040. "vld4.u8 {d0-d3}, [%1], %10 \n"
  3041. "pld [%2, #256] \n"
  3042. "vld4.u8 {d4-d7}, [%2], %10 \n"
  3043. "pld [%1, #256] \n"
  3044. "vld4.u8 {d8-d11}, [%1], %10 \n"
  3045. "vtrn.u8 q2, q0 \n" // _src01t_r
  3046. "vtrn.u8 q3, q1 \n"
  3047. "pld [%2, #256] \n"
  3048. "vld4.u8 {d12-d15}, [%2], %10\n"
  3049. "pld [%1, #256] \n"
  3050. "vld4.u8 {d16-d19}, [%1], %10\n"
  3051. "vtrn.u8 q6, q4 \n" // _src23t_r
  3052. "vtrn.u8 q7, q5 \n"
  3053. "pld [%2, #256] \n"
  3054. "vld4.u8 {d20-d23}, [%2], %10\n"
  3055. "pld [%1, #256] \n"
  3056. "vld4.u8 {d24-d27}, [%1], %10\n"
  3057. "vtrn.u8 q10, q8 \n" // _src45t_r
  3058. "vtrn.u8 q11, q9 \n"
  3059. "pld [%2, #256] \n"
  3060. "vld4.u8 {d28-d31}, [%2], %10\n"
  3061. "vtrn.u8 q14, q12 \n" // _src67t_r
  3062. "vtrn.u8 q15, q13 \n"
  3063. "sub %1, %1, %10, lsl #2 \n" // restore src0
  3064. "vtrn.u16 q4, q0 \n" // _src02tt_r
  3065. "vtrn.u16 q5, q1 \n"
  3066. "sub %2, %2, %10, lsl #2 \n" // restore src1
  3067. "vtrn.u16 q6, q2 \n" // _src13tt_r
  3068. "vtrn.u16 q7, q3 \n"
  3069. "add %1, #32 \n" // src0 += 32
  3070. "vtrn.u16 q12, q8 \n" // _src46tt_r
  3071. "vtrn.u16 q13, q9 \n"
  3072. "add %2, #32 \n" // src1 += 32
  3073. "vtrn.u16 q14, q10 \n" // _src57tt_r
  3074. "vtrn.u16 q15, q11 \n"
  3075. "vtrn.u32 q12, q4 \n" // _src26ttt_r
  3076. "vtrn.u32 q13, q5 \n"
  3077. "vtrn.u32 q14, q6 \n" // _src37ttt_r
  3078. "vst4.u8 {d24-d27}, [%4], %11\n"
  3079. "vtrn.u32 q15, q7 \n"
  3080. "vtrn.u32 q8, q0 \n" // _src04ttt_r
  3081. "vst4.u8 {d28-d31}, [%3], %11\n"
  3082. "vtrn.u32 q9, q1 \n"
  3083. "vtrn.u32 q10, q2 \n" // _src15ttt_r
  3084. "vst4.u8 {d16-d19}, [%4], %11\n"
  3085. "vtrn.u32 q11, q3 \n"
  3086. "subs %0, #1 \n"
  3087. "vst4.u8 {d8-d11}, [%4], %11 \n"
  3088. "vst4.u8 {d20-d23}, [%3], %11\n"
  3089. "vst4.u8 {d12-d15}, [%3], %11\n"
  3090. "vst4.u8 {d0-d3}, [%4], %11 \n"
  3091. "vst4.u8 {d4-d7}, [%3], %11 \n"
  3092. "bne 0b \n"
  3093. : "=r"(nn), // %0
  3094. "=r"(src0), // %1
  3095. "=r"(src1), // %2
  3096. "=r"(dst0), // %3
  3097. "=r"(dst1) // %4
  3098. : "0"(nn),
  3099. "1"(src0),
  3100. "2"(src1),
  3101. "3"(dst0),
  3102. "4"(dst1),
  3103. "r"(src_step), // %10
  3104. "r"(dst_step) // %11
  3105. : "cc", "memory", "q0", "q1", "q2", "q3", "q4", "q5", "q6", "q7", "q8", "q9", "q10", "q11", "q12", "q13", "q14", "q15");
  3106. }
  3107. #endif // __aarch64__
  3108. for (; remain > 0; remain--)
  3109. {
  3110. dst0[0] = src1[0 + 3 * src_step];
  3111. dst0[1] = src1[1 + 3 * src_step];
  3112. dst0[2] = src1[2 + 3 * src_step];
  3113. dst0[3] = src1[3 + 3 * src_step];
  3114. dst0[4] = src0[0 + 3 * src_step];
  3115. dst0[5] = src0[1 + 3 * src_step];
  3116. dst0[6] = src0[2 + 3 * src_step];
  3117. dst0[7] = src0[3 + 3 * src_step];
  3118. dst0[8] = src1[0 + 2 * src_step];
  3119. dst0[9] = src1[1 + 2 * src_step];
  3120. dst0[10] = src1[2 + 2 * src_step];
  3121. dst0[11] = src1[3 + 2 * src_step];
  3122. dst0[12] = src0[0 + 2 * src_step];
  3123. dst0[13] = src0[1 + 2 * src_step];
  3124. dst0[14] = src0[2 + 2 * src_step];
  3125. dst0[15] = src0[3 + 2 * src_step];
  3126. dst0[16] = src1[0 + src_step];
  3127. dst0[17] = src1[1 + src_step];
  3128. dst0[18] = src1[2 + src_step];
  3129. dst0[19] = src1[3 + src_step];
  3130. dst0[20] = src0[0 + src_step];
  3131. dst0[21] = src0[1 + src_step];
  3132. dst0[22] = src0[2 + src_step];
  3133. dst0[23] = src0[3 + src_step];
  3134. dst0[24] = src1[0];
  3135. dst0[25] = src1[1];
  3136. dst0[26] = src1[2];
  3137. dst0[27] = src1[3];
  3138. dst0[28] = src0[0];
  3139. dst0[29] = src0[1];
  3140. dst0[30] = src0[2];
  3141. dst0[31] = src0[3];
  3142. src0 += 4;
  3143. src1 += 4;
  3144. dst0 += stride;
  3145. }
  3146. src0 += srcwgap + 7 * srcstride;
  3147. }
  3148. #endif // __ARM_NEON
  3149. for (; y < srch; y++)
  3150. {
  3151. unsigned char* dst0 = dstend - y * 4 - 4;
  3152. int x = 0;
  3153. for (; x < srcw; x++)
  3154. {
  3155. dst0[0] = src0[0];
  3156. dst0[1] = src0[1];
  3157. dst0[2] = src0[2];
  3158. dst0[3] = src0[3];
  3159. src0 += 4;
  3160. dst0 += stride;
  3161. }
  3162. src0 += srcwgap;
  3163. }
  3164. }
  3165. static void kanna_rotate_7_c1(const unsigned char* src, int srcw, int srch, int srcstride, unsigned char* dst, int w, int h, int stride)
  3166. {
  3167. const int srcwgap = srcstride - srcw;
  3168. // point to the last dst pixel
  3169. unsigned char* dstend = dst + stride * (h - 1) + w;
  3170. const unsigned char* src0 = src;
  3171. int y = 0;
  3172. #if __ARM_NEON
  3173. for (; y + 7 < srch; y += 8)
  3174. {
  3175. const unsigned char* src1 = src0 + srcstride;
  3176. unsigned char* dst6 = dstend - y - 8 - stride;
  3177. unsigned char* dst7 = dstend - y - 8;
  3178. int src_step = 2 * srcstride;
  3179. int dst_step = -2 * stride;
  3180. int nn = srcw >> 3;
  3181. int remain = srcw - (nn << 3);
  3182. #if !NCNN_GNU_INLINE_ASM || __aarch64__
  3183. for (; nn > 0; nn--)
  3184. {
  3185. uint8x8_t _src0 = vld1_u8(src0);
  3186. uint8x8_t _src1 = vld1_u8(src1);
  3187. uint8x8_t _src2 = vld1_u8(src0 + src_step);
  3188. uint8x8_t _src3 = vld1_u8(src1 + src_step);
  3189. uint8x8_t _src4 = vld1_u8(src0 + 2 * src_step);
  3190. uint8x8_t _src5 = vld1_u8(src1 + 2 * src_step);
  3191. uint8x8_t _src6 = vld1_u8(src0 + 3 * src_step);
  3192. uint8x8_t _src7 = vld1_u8(src1 + 3 * src_step);
  3193. uint8x8x2_t _src01t_r = vtrn_u8(_src1, _src0);
  3194. uint8x8x2_t _src23t_r = vtrn_u8(_src3, _src2);
  3195. uint8x8x2_t _src45t_r = vtrn_u8(_src5, _src4);
  3196. uint8x8x2_t _src67t_r = vtrn_u8(_src7, _src6);
  3197. uint16x4x2_t _src02tt_r = vtrn_u16(vreinterpret_u16_u8(_src23t_r.val[1]), vreinterpret_u16_u8(_src01t_r.val[1]));
  3198. uint16x4x2_t _src13tt_r = vtrn_u16(vreinterpret_u16_u8(_src23t_r.val[0]), vreinterpret_u16_u8(_src01t_r.val[0]));
  3199. uint16x4x2_t _src46tt_r = vtrn_u16(vreinterpret_u16_u8(_src67t_r.val[1]), vreinterpret_u16_u8(_src45t_r.val[1]));
  3200. uint16x4x2_t _src57tt_r = vtrn_u16(vreinterpret_u16_u8(_src67t_r.val[0]), vreinterpret_u16_u8(_src45t_r.val[0]));
  3201. uint32x2x2_t _src04ttt_r = vtrn_u32(vreinterpret_u32_u16(_src46tt_r.val[1]), vreinterpret_u32_u16(_src02tt_r.val[1]));
  3202. uint32x2x2_t _src15ttt_r = vtrn_u32(vreinterpret_u32_u16(_src57tt_r.val[1]), vreinterpret_u32_u16(_src13tt_r.val[1]));
  3203. uint32x2x2_t _src26ttt_r = vtrn_u32(vreinterpret_u32_u16(_src46tt_r.val[0]), vreinterpret_u32_u16(_src02tt_r.val[0]));
  3204. uint32x2x2_t _src37ttt_r = vtrn_u32(vreinterpret_u32_u16(_src57tt_r.val[0]), vreinterpret_u32_u16(_src13tt_r.val[0]));
  3205. uint8x8_t _dst0 = vreinterpret_u8_u32(_src04ttt_r.val[1]);
  3206. uint8x8_t _dst1 = vreinterpret_u8_u32(_src15ttt_r.val[1]);
  3207. uint8x8_t _dst2 = vreinterpret_u8_u32(_src26ttt_r.val[1]);
  3208. uint8x8_t _dst3 = vreinterpret_u8_u32(_src37ttt_r.val[1]);
  3209. uint8x8_t _dst4 = vreinterpret_u8_u32(_src04ttt_r.val[0]);
  3210. uint8x8_t _dst5 = vreinterpret_u8_u32(_src15ttt_r.val[0]);
  3211. uint8x8_t _dst6 = vreinterpret_u8_u32(_src26ttt_r.val[0]);
  3212. uint8x8_t _dst7 = vreinterpret_u8_u32(_src37ttt_r.val[0]);
  3213. vst1_u8(dst7, _dst7);
  3214. vst1_u8(dst6, _dst6);
  3215. vst1_u8(dst7 + dst_step, _dst5);
  3216. vst1_u8(dst6 + dst_step, _dst4);
  3217. vst1_u8(dst7 + 2 * dst_step, _dst3);
  3218. vst1_u8(dst6 + 2 * dst_step, _dst2);
  3219. vst1_u8(dst7 + 3 * dst_step, _dst1);
  3220. vst1_u8(dst6 + 3 * dst_step, _dst0);
  3221. src0 += 8;
  3222. src1 += 8;
  3223. dst7 += 4 * dst_step;
  3224. dst6 += 4 * dst_step;
  3225. }
  3226. #else
  3227. if (nn > 0)
  3228. {
  3229. asm volatile(
  3230. "0: \n"
  3231. "pld [%1, #64] \n"
  3232. "vld1.u8 {d0}, [%1], %10 \n"
  3233. "pld [%2, #64] \n"
  3234. "vld1.u8 {d1}, [%2], %10 \n"
  3235. "pld [%1, #64] \n"
  3236. "vld1.u8 {d2}, [%1], %10 \n"
  3237. "vtrn.u8 d1, d0 \n" // _src01t_r
  3238. "pld [%2, #64] \n"
  3239. "vld1.u8 {d3}, [%2], %10 \n"
  3240. "pld [%1, #64] \n"
  3241. "vld1.u8 {d4}, [%1], %10 \n"
  3242. "vtrn.u8 d3, d2 \n" // _src23t_r
  3243. "pld [%2, #64] \n"
  3244. "vld1.u8 {d5}, [%2], %10 \n"
  3245. "pld [%1, #64] \n"
  3246. "vld1.u8 {d6}, [%1], %10 \n"
  3247. "vtrn.u8 d5, d4 \n" // _src45t_r
  3248. "pld [%2, #64] \n"
  3249. "vld1.u8 {d7}, [%2], %10 \n"
  3250. "vtrn.u8 d7, d6 \n" // _src67t_r
  3251. "sub %1, %1, %10, lsl #2 \n" // restore src0
  3252. "vtrn.u16 q1, q0 \n" // _src02tt_r _src13tt_r
  3253. "sub %2, %2, %10, lsl #2 \n" // restore src1
  3254. "vtrn.u16 q3, q2 \n" // _src46tt_r _src57tt_r
  3255. "add %1, #8 \n" // src0 += 8
  3256. "vtrn.u32 q3, q1 \n" // _src26ttt_r _src37ttt_r
  3257. "add %2, #8 \n" // src1 += 8
  3258. "vtrn.u32 q2, q0 \n" // _src04ttt_r _src15ttt_r
  3259. "vst1.u8 {d6}, [%4], %11 \n"
  3260. "vst1.u8 {d7}, [%3], %11 \n"
  3261. "subs %0, #1 \n"
  3262. "vst1.u8 {d4}, [%4], %11 \n"
  3263. "vst1.u8 {d5}, [%3], %11 \n"
  3264. "vst1.u8 {d2}, [%4], %11 \n"
  3265. "vst1.u8 {d3}, [%3], %11 \n"
  3266. "vst1.u8 {d0}, [%4], %11 \n"
  3267. "vst1.u8 {d1}, [%3], %11 \n"
  3268. "bne 0b \n"
  3269. : "=r"(nn), // %0
  3270. "=r"(src0), // %1
  3271. "=r"(src1), // %2
  3272. "=r"(dst7), // %3
  3273. "=r"(dst6) // %4
  3274. : "0"(nn),
  3275. "1"(src0),
  3276. "2"(src1),
  3277. "3"(dst7),
  3278. "4"(dst6),
  3279. "r"(src_step), // %10
  3280. "r"(dst_step) // %11
  3281. : "cc", "memory", "q0", "q1", "q2", "q3");
  3282. }
  3283. #endif // __aarch64__
  3284. for (; remain > 0; remain--)
  3285. {
  3286. dst7[0] = src1[0 + 3 * src_step];
  3287. dst7[1] = src0[0 + 3 * src_step];
  3288. dst7[2] = src1[0 + 2 * src_step];
  3289. dst7[3] = src0[0 + 2 * src_step];
  3290. dst7[4] = src1[0 + src_step];
  3291. dst7[5] = src0[0 + src_step];
  3292. dst7[6] = src1[0];
  3293. dst7[7] = src0[0];
  3294. src0 += 1;
  3295. src1 += 1;
  3296. dst7 -= stride;
  3297. }
  3298. src0 += srcwgap + 7 * srcstride;
  3299. }
  3300. #endif // __ARM_NEON
  3301. for (; y < srch; y++)
  3302. {
  3303. unsigned char* dst0 = dstend - y - 1;
  3304. int x = 0;
  3305. for (; x < srcw; x++)
  3306. {
  3307. *dst0 = *src0;
  3308. src0 += 1;
  3309. dst0 -= stride;
  3310. }
  3311. src0 += srcwgap;
  3312. }
  3313. }
  3314. static void kanna_rotate_7_c2(const unsigned char* src, int srcw, int srch, int srcstride, unsigned char* dst, int w, int h, int stride)
  3315. {
  3316. const int srcwgap = srcstride - srcw * 2;
  3317. // point to the last dst pixel
  3318. unsigned char* dstend = dst + stride * (h - 1) + w * 2;
  3319. const unsigned char* src0 = src;
  3320. int y = 0;
  3321. #if __ARM_NEON
  3322. for (; y + 7 < srch; y += 8)
  3323. {
  3324. const unsigned char* src1 = src0 + srcstride;
  3325. unsigned char* dst6 = dstend - y * 2 - 8 * 2 - stride;
  3326. unsigned char* dst7 = dstend - y * 2 - 8 * 2;
  3327. int src_step = 2 * srcstride;
  3328. int dst_step = -2 * stride;
  3329. int nn = srcw >> 3;
  3330. int remain = srcw - (nn << 3);
  3331. #if !NCNN_GNU_INLINE_ASM || __aarch64__
  3332. for (; nn > 0; nn--)
  3333. {
  3334. uint8x8x2_t _src0 = vld2_u8(src0);
  3335. uint8x8x2_t _src1 = vld2_u8(src1);
  3336. uint8x8x2_t _src2 = vld2_u8(src0 + src_step);
  3337. uint8x8x2_t _src3 = vld2_u8(src1 + src_step);
  3338. uint8x8x2_t _src4 = vld2_u8(src0 + 2 * src_step);
  3339. uint8x8x2_t _src5 = vld2_u8(src1 + 2 * src_step);
  3340. uint8x8x2_t _src6 = vld2_u8(src0 + 3 * src_step);
  3341. uint8x8x2_t _src7 = vld2_u8(src1 + 3 * src_step);
  3342. uint8x8x2_t _src01t_r = vtrn_u8(_src1.val[0], _src0.val[0]);
  3343. uint8x8x2_t _src23t_r = vtrn_u8(_src3.val[0], _src2.val[0]);
  3344. uint8x8x2_t _src45t_r = vtrn_u8(_src5.val[0], _src4.val[0]);
  3345. uint8x8x2_t _src67t_r = vtrn_u8(_src7.val[0], _src6.val[0]);
  3346. uint8x8x2_t _src01t_g = vtrn_u8(_src1.val[1], _src0.val[1]);
  3347. uint8x8x2_t _src23t_g = vtrn_u8(_src3.val[1], _src2.val[1]);
  3348. uint8x8x2_t _src45t_g = vtrn_u8(_src5.val[1], _src4.val[1]);
  3349. uint8x8x2_t _src67t_g = vtrn_u8(_src7.val[1], _src6.val[1]);
  3350. uint16x4x2_t _src02tt_r = vtrn_u16(vreinterpret_u16_u8(_src23t_r.val[1]), vreinterpret_u16_u8(_src01t_r.val[1]));
  3351. uint16x4x2_t _src13tt_r = vtrn_u16(vreinterpret_u16_u8(_src23t_r.val[0]), vreinterpret_u16_u8(_src01t_r.val[0]));
  3352. uint16x4x2_t _src46tt_r = vtrn_u16(vreinterpret_u16_u8(_src67t_r.val[1]), vreinterpret_u16_u8(_src45t_r.val[1]));
  3353. uint16x4x2_t _src57tt_r = vtrn_u16(vreinterpret_u16_u8(_src67t_r.val[0]), vreinterpret_u16_u8(_src45t_r.val[0]));
  3354. uint16x4x2_t _src02tt_g = vtrn_u16(vreinterpret_u16_u8(_src23t_g.val[1]), vreinterpret_u16_u8(_src01t_g.val[1]));
  3355. uint16x4x2_t _src13tt_g = vtrn_u16(vreinterpret_u16_u8(_src23t_g.val[0]), vreinterpret_u16_u8(_src01t_g.val[0]));
  3356. uint16x4x2_t _src46tt_g = vtrn_u16(vreinterpret_u16_u8(_src67t_g.val[1]), vreinterpret_u16_u8(_src45t_g.val[1]));
  3357. uint16x4x2_t _src57tt_g = vtrn_u16(vreinterpret_u16_u8(_src67t_g.val[0]), vreinterpret_u16_u8(_src45t_g.val[0]));
  3358. uint32x2x2_t _src04ttt_r = vtrn_u32(vreinterpret_u32_u16(_src46tt_r.val[1]), vreinterpret_u32_u16(_src02tt_r.val[1]));
  3359. uint32x2x2_t _src15ttt_r = vtrn_u32(vreinterpret_u32_u16(_src57tt_r.val[1]), vreinterpret_u32_u16(_src13tt_r.val[1]));
  3360. uint32x2x2_t _src26ttt_r = vtrn_u32(vreinterpret_u32_u16(_src46tt_r.val[0]), vreinterpret_u32_u16(_src02tt_r.val[0]));
  3361. uint32x2x2_t _src37ttt_r = vtrn_u32(vreinterpret_u32_u16(_src57tt_r.val[0]), vreinterpret_u32_u16(_src13tt_r.val[0]));
  3362. uint32x2x2_t _src04ttt_g = vtrn_u32(vreinterpret_u32_u16(_src46tt_g.val[1]), vreinterpret_u32_u16(_src02tt_g.val[1]));
  3363. uint32x2x2_t _src15ttt_g = vtrn_u32(vreinterpret_u32_u16(_src57tt_g.val[1]), vreinterpret_u32_u16(_src13tt_g.val[1]));
  3364. uint32x2x2_t _src26ttt_g = vtrn_u32(vreinterpret_u32_u16(_src46tt_g.val[0]), vreinterpret_u32_u16(_src02tt_g.val[0]));
  3365. uint32x2x2_t _src37ttt_g = vtrn_u32(vreinterpret_u32_u16(_src57tt_g.val[0]), vreinterpret_u32_u16(_src13tt_g.val[0]));
  3366. uint8x8x2_t _dst0;
  3367. uint8x8x2_t _dst1;
  3368. uint8x8x2_t _dst2;
  3369. uint8x8x2_t _dst3;
  3370. uint8x8x2_t _dst4;
  3371. uint8x8x2_t _dst5;
  3372. uint8x8x2_t _dst6;
  3373. uint8x8x2_t _dst7;
  3374. _dst0.val[0] = vreinterpret_u8_u32(_src04ttt_r.val[1]);
  3375. _dst1.val[0] = vreinterpret_u8_u32(_src15ttt_r.val[1]);
  3376. _dst2.val[0] = vreinterpret_u8_u32(_src26ttt_r.val[1]);
  3377. _dst3.val[0] = vreinterpret_u8_u32(_src37ttt_r.val[1]);
  3378. _dst4.val[0] = vreinterpret_u8_u32(_src04ttt_r.val[0]);
  3379. _dst5.val[0] = vreinterpret_u8_u32(_src15ttt_r.val[0]);
  3380. _dst6.val[0] = vreinterpret_u8_u32(_src26ttt_r.val[0]);
  3381. _dst7.val[0] = vreinterpret_u8_u32(_src37ttt_r.val[0]);
  3382. _dst0.val[1] = vreinterpret_u8_u32(_src04ttt_g.val[1]);
  3383. _dst1.val[1] = vreinterpret_u8_u32(_src15ttt_g.val[1]);
  3384. _dst2.val[1] = vreinterpret_u8_u32(_src26ttt_g.val[1]);
  3385. _dst3.val[1] = vreinterpret_u8_u32(_src37ttt_g.val[1]);
  3386. _dst4.val[1] = vreinterpret_u8_u32(_src04ttt_g.val[0]);
  3387. _dst5.val[1] = vreinterpret_u8_u32(_src15ttt_g.val[0]);
  3388. _dst6.val[1] = vreinterpret_u8_u32(_src26ttt_g.val[0]);
  3389. _dst7.val[1] = vreinterpret_u8_u32(_src37ttt_g.val[0]);
  3390. vst2_u8(dst7, _dst7);
  3391. vst2_u8(dst6, _dst6);
  3392. vst2_u8(dst7 + dst_step, _dst5);
  3393. vst2_u8(dst6 + dst_step, _dst4);
  3394. vst2_u8(dst7 + 2 * dst_step, _dst3);
  3395. vst2_u8(dst6 + 2 * dst_step, _dst2);
  3396. vst2_u8(dst7 + 3 * dst_step, _dst1);
  3397. vst2_u8(dst6 + 3 * dst_step, _dst0);
  3398. src0 += 2 * 8;
  3399. src1 += 2 * 8;
  3400. dst7 += 4 * dst_step;
  3401. dst6 += 4 * dst_step;
  3402. }
  3403. #else
  3404. if (nn > 0)
  3405. {
  3406. asm volatile(
  3407. "0: \n"
  3408. "pld [%1, #128] \n"
  3409. "vld2.u8 {d0-d1}, [%1], %10 \n"
  3410. "pld [%2, #128] \n"
  3411. "vld2.u8 {d2-d3}, [%2], %10 \n"
  3412. "pld [%1, #128] \n"
  3413. "vld2.u8 {d4-d5}, [%1], %10 \n"
  3414. "vtrn.u8 q1, q0 \n" // _src01t_r
  3415. "pld [%2, #128] \n"
  3416. "vld2.u8 {d6-d7}, [%2], %10 \n"
  3417. "pld [%1, #128] \n"
  3418. "vld2.u8 {d16-d17}, [%1], %10\n"
  3419. "vtrn.u8 q3, q2 \n" // _src23t_r
  3420. "pld [%2, #128] \n"
  3421. "vld2.u8 {d18-d19}, [%2], %10\n"
  3422. "pld [%1, #128] \n"
  3423. "vld2.u8 {d20-d21}, [%1], %10\n"
  3424. "vtrn.u8 q9, q8 \n" // _src45t_r
  3425. "pld [%2, #128] \n"
  3426. "vld2.u8 {d22-d23}, [%2], %10\n"
  3427. "vtrn.u8 q11, q10 \n" // _src67t_r
  3428. "sub %1, %1, %10, lsl #2 \n" // restore src0
  3429. "vtrn.u16 q2, q0 \n" // _src02tt_r
  3430. "sub %2, %2, %10, lsl #2 \n" // restore src1
  3431. "vtrn.u16 q3, q1 \n" // _src13tt_r
  3432. "add %1, #16 \n" // src0 += 16
  3433. "vtrn.u16 q10, q8 \n" // _src46tt_r
  3434. "add %2, #16 \n" // src1 += 16
  3435. "vtrn.u16 q11, q9 \n" // _src57tt_r
  3436. "vtrn.u32 q10, q2 \n" // _src26ttt_r
  3437. "vtrn.u32 q11, q3 \n" // _src37ttt_r
  3438. "vst2.u8 {d20-d21}, [%4], %11\n"
  3439. "vtrn.u32 q8, q0 \n" // _src04ttt_r
  3440. "vst2.u8 {d22-d23}, [%3], %11\n"
  3441. "vtrn.u32 q9, q1 \n" // _src15ttt_r
  3442. "vst2.u8 {d16-d17}, [%4], %11\n"
  3443. "subs %0, #1 \n"
  3444. "vst2.u8 {d4-d5}, [%4], %11 \n"
  3445. "vst2.u8 {d18-d19}, [%3], %11\n"
  3446. "vst2.u8 {d6-d7}, [%3], %11 \n"
  3447. "vst2.u8 {d0-d1}, [%4], %11 \n"
  3448. "vst2.u8 {d2-d3}, [%3], %11 \n"
  3449. "bne 0b \n"
  3450. : "=r"(nn), // %0
  3451. "=r"(src0), // %1
  3452. "=r"(src1), // %2
  3453. "=r"(dst7), // %3
  3454. "=r"(dst6) // %4
  3455. : "0"(nn),
  3456. "1"(src0),
  3457. "2"(src1),
  3458. "3"(dst7),
  3459. "4"(dst6),
  3460. "r"(src_step), // %10
  3461. "r"(dst_step) // %11
  3462. : "cc", "memory", "q0", "q1", "q2", "q3", "q8", "q9", "q10", "q11");
  3463. }
  3464. #endif // __aarch64__
  3465. for (; remain > 0; remain--)
  3466. {
  3467. dst7[0] = src1[0 + 3 * src_step];
  3468. dst7[1] = src1[1 + 3 * src_step];
  3469. dst7[2] = src0[0 + 3 * src_step];
  3470. dst7[3] = src0[1 + 3 * src_step];
  3471. dst7[4] = src1[0 + 2 * src_step];
  3472. dst7[5] = src1[1 + 2 * src_step];
  3473. dst7[6] = src0[0 + 2 * src_step];
  3474. dst7[7] = src0[1 + 2 * src_step];
  3475. dst7[8] = src1[0 + src_step];
  3476. dst7[9] = src1[1 + src_step];
  3477. dst7[10] = src0[0 + src_step];
  3478. dst7[11] = src0[1 + src_step];
  3479. dst7[12] = src1[0];
  3480. dst7[13] = src1[1];
  3481. dst7[14] = src0[0];
  3482. dst7[15] = src0[1];
  3483. src0 += 2;
  3484. src1 += 2;
  3485. dst7 -= stride;
  3486. }
  3487. src0 += srcwgap + 7 * srcstride;
  3488. }
  3489. #endif // __ARM_NEON
  3490. for (; y < srch; y++)
  3491. {
  3492. unsigned char* dst0 = dstend - y * 2 - 2;
  3493. int x = 0;
  3494. for (; x < srcw; x++)
  3495. {
  3496. dst0[0] = src0[0];
  3497. dst0[1] = src0[1];
  3498. src0 += 2;
  3499. dst0 -= stride;
  3500. }
  3501. src0 += srcwgap;
  3502. }
  3503. }
  3504. static void kanna_rotate_7_c3(const unsigned char* src, int srcw, int srch, int srcstride, unsigned char* dst, int w, int h, int stride)
  3505. {
  3506. const int srcwgap = srcstride - srcw * 3;
  3507. // point to the last dst pixel
  3508. unsigned char* dstend = dst + stride * (h - 1) + w * 3;
  3509. const unsigned char* src0 = src;
  3510. int y = 0;
  3511. #if __ARM_NEON
  3512. for (; y + 7 < srch; y += 8)
  3513. {
  3514. const unsigned char* src1 = src0 + srcstride;
  3515. unsigned char* dst6 = dstend - y * 3 - 8 * 3 - stride;
  3516. unsigned char* dst7 = dstend - y * 3 - 8 * 3;
  3517. int src_step = 2 * srcstride;
  3518. int dst_step = -2 * stride;
  3519. int nn = srcw >> 3;
  3520. int remain = srcw - (nn << 3);
  3521. #if !NCNN_GNU_INLINE_ASM || __aarch64__
  3522. for (; nn > 0; nn--)
  3523. {
  3524. uint8x8x3_t _src0 = vld3_u8(src0);
  3525. uint8x8x3_t _src1 = vld3_u8(src1);
  3526. uint8x8x3_t _src2 = vld3_u8(src0 + src_step);
  3527. uint8x8x3_t _src3 = vld3_u8(src1 + src_step);
  3528. uint8x8x3_t _src4 = vld3_u8(src0 + 2 * src_step);
  3529. uint8x8x3_t _src5 = vld3_u8(src1 + 2 * src_step);
  3530. uint8x8x3_t _src6 = vld3_u8(src0 + 3 * src_step);
  3531. uint8x8x3_t _src7 = vld3_u8(src1 + 3 * src_step);
  3532. uint8x8x2_t _src01t_r = vtrn_u8(_src1.val[0], _src0.val[0]);
  3533. uint8x8x2_t _src23t_r = vtrn_u8(_src3.val[0], _src2.val[0]);
  3534. uint8x8x2_t _src45t_r = vtrn_u8(_src5.val[0], _src4.val[0]);
  3535. uint8x8x2_t _src67t_r = vtrn_u8(_src7.val[0], _src6.val[0]);
  3536. uint8x8x2_t _src01t_g = vtrn_u8(_src1.val[1], _src0.val[1]);
  3537. uint8x8x2_t _src23t_g = vtrn_u8(_src3.val[1], _src2.val[1]);
  3538. uint8x8x2_t _src45t_g = vtrn_u8(_src5.val[1], _src4.val[1]);
  3539. uint8x8x2_t _src67t_g = vtrn_u8(_src7.val[1], _src6.val[1]);
  3540. uint8x8x2_t _src01t_b = vtrn_u8(_src1.val[2], _src0.val[2]);
  3541. uint8x8x2_t _src23t_b = vtrn_u8(_src3.val[2], _src2.val[2]);
  3542. uint8x8x2_t _src45t_b = vtrn_u8(_src5.val[2], _src4.val[2]);
  3543. uint8x8x2_t _src67t_b = vtrn_u8(_src7.val[2], _src6.val[2]);
  3544. uint16x4x2_t _src02tt_r = vtrn_u16(vreinterpret_u16_u8(_src23t_r.val[1]), vreinterpret_u16_u8(_src01t_r.val[1]));
  3545. uint16x4x2_t _src13tt_r = vtrn_u16(vreinterpret_u16_u8(_src23t_r.val[0]), vreinterpret_u16_u8(_src01t_r.val[0]));
  3546. uint16x4x2_t _src46tt_r = vtrn_u16(vreinterpret_u16_u8(_src67t_r.val[1]), vreinterpret_u16_u8(_src45t_r.val[1]));
  3547. uint16x4x2_t _src57tt_r = vtrn_u16(vreinterpret_u16_u8(_src67t_r.val[0]), vreinterpret_u16_u8(_src45t_r.val[0]));
  3548. uint16x4x2_t _src02tt_g = vtrn_u16(vreinterpret_u16_u8(_src23t_g.val[1]), vreinterpret_u16_u8(_src01t_g.val[1]));
  3549. uint16x4x2_t _src13tt_g = vtrn_u16(vreinterpret_u16_u8(_src23t_g.val[0]), vreinterpret_u16_u8(_src01t_g.val[0]));
  3550. uint16x4x2_t _src46tt_g = vtrn_u16(vreinterpret_u16_u8(_src67t_g.val[1]), vreinterpret_u16_u8(_src45t_g.val[1]));
  3551. uint16x4x2_t _src57tt_g = vtrn_u16(vreinterpret_u16_u8(_src67t_g.val[0]), vreinterpret_u16_u8(_src45t_g.val[0]));
  3552. uint16x4x2_t _src02tt_b = vtrn_u16(vreinterpret_u16_u8(_src23t_b.val[1]), vreinterpret_u16_u8(_src01t_b.val[1]));
  3553. uint16x4x2_t _src13tt_b = vtrn_u16(vreinterpret_u16_u8(_src23t_b.val[0]), vreinterpret_u16_u8(_src01t_b.val[0]));
  3554. uint16x4x2_t _src46tt_b = vtrn_u16(vreinterpret_u16_u8(_src67t_b.val[1]), vreinterpret_u16_u8(_src45t_b.val[1]));
  3555. uint16x4x2_t _src57tt_b = vtrn_u16(vreinterpret_u16_u8(_src67t_b.val[0]), vreinterpret_u16_u8(_src45t_b.val[0]));
  3556. uint32x2x2_t _src04ttt_r = vtrn_u32(vreinterpret_u32_u16(_src46tt_r.val[1]), vreinterpret_u32_u16(_src02tt_r.val[1]));
  3557. uint32x2x2_t _src15ttt_r = vtrn_u32(vreinterpret_u32_u16(_src57tt_r.val[1]), vreinterpret_u32_u16(_src13tt_r.val[1]));
  3558. uint32x2x2_t _src26ttt_r = vtrn_u32(vreinterpret_u32_u16(_src46tt_r.val[0]), vreinterpret_u32_u16(_src02tt_r.val[0]));
  3559. uint32x2x2_t _src37ttt_r = vtrn_u32(vreinterpret_u32_u16(_src57tt_r.val[0]), vreinterpret_u32_u16(_src13tt_r.val[0]));
  3560. uint32x2x2_t _src04ttt_g = vtrn_u32(vreinterpret_u32_u16(_src46tt_g.val[1]), vreinterpret_u32_u16(_src02tt_g.val[1]));
  3561. uint32x2x2_t _src15ttt_g = vtrn_u32(vreinterpret_u32_u16(_src57tt_g.val[1]), vreinterpret_u32_u16(_src13tt_g.val[1]));
  3562. uint32x2x2_t _src26ttt_g = vtrn_u32(vreinterpret_u32_u16(_src46tt_g.val[0]), vreinterpret_u32_u16(_src02tt_g.val[0]));
  3563. uint32x2x2_t _src37ttt_g = vtrn_u32(vreinterpret_u32_u16(_src57tt_g.val[0]), vreinterpret_u32_u16(_src13tt_g.val[0]));
  3564. uint32x2x2_t _src04ttt_b = vtrn_u32(vreinterpret_u32_u16(_src46tt_b.val[1]), vreinterpret_u32_u16(_src02tt_b.val[1]));
  3565. uint32x2x2_t _src15ttt_b = vtrn_u32(vreinterpret_u32_u16(_src57tt_b.val[1]), vreinterpret_u32_u16(_src13tt_b.val[1]));
  3566. uint32x2x2_t _src26ttt_b = vtrn_u32(vreinterpret_u32_u16(_src46tt_b.val[0]), vreinterpret_u32_u16(_src02tt_b.val[0]));
  3567. uint32x2x2_t _src37ttt_b = vtrn_u32(vreinterpret_u32_u16(_src57tt_b.val[0]), vreinterpret_u32_u16(_src13tt_b.val[0]));
  3568. uint8x8x3_t _dst0;
  3569. uint8x8x3_t _dst1;
  3570. uint8x8x3_t _dst2;
  3571. uint8x8x3_t _dst3;
  3572. uint8x8x3_t _dst4;
  3573. uint8x8x3_t _dst5;
  3574. uint8x8x3_t _dst6;
  3575. uint8x8x3_t _dst7;
  3576. _dst0.val[0] = vreinterpret_u8_u32(_src04ttt_r.val[1]);
  3577. _dst1.val[0] = vreinterpret_u8_u32(_src15ttt_r.val[1]);
  3578. _dst2.val[0] = vreinterpret_u8_u32(_src26ttt_r.val[1]);
  3579. _dst3.val[0] = vreinterpret_u8_u32(_src37ttt_r.val[1]);
  3580. _dst4.val[0] = vreinterpret_u8_u32(_src04ttt_r.val[0]);
  3581. _dst5.val[0] = vreinterpret_u8_u32(_src15ttt_r.val[0]);
  3582. _dst6.val[0] = vreinterpret_u8_u32(_src26ttt_r.val[0]);
  3583. _dst7.val[0] = vreinterpret_u8_u32(_src37ttt_r.val[0]);
  3584. _dst0.val[1] = vreinterpret_u8_u32(_src04ttt_g.val[1]);
  3585. _dst1.val[1] = vreinterpret_u8_u32(_src15ttt_g.val[1]);
  3586. _dst2.val[1] = vreinterpret_u8_u32(_src26ttt_g.val[1]);
  3587. _dst3.val[1] = vreinterpret_u8_u32(_src37ttt_g.val[1]);
  3588. _dst4.val[1] = vreinterpret_u8_u32(_src04ttt_g.val[0]);
  3589. _dst5.val[1] = vreinterpret_u8_u32(_src15ttt_g.val[0]);
  3590. _dst6.val[1] = vreinterpret_u8_u32(_src26ttt_g.val[0]);
  3591. _dst7.val[1] = vreinterpret_u8_u32(_src37ttt_g.val[0]);
  3592. _dst0.val[2] = vreinterpret_u8_u32(_src04ttt_b.val[1]);
  3593. _dst1.val[2] = vreinterpret_u8_u32(_src15ttt_b.val[1]);
  3594. _dst2.val[2] = vreinterpret_u8_u32(_src26ttt_b.val[1]);
  3595. _dst3.val[2] = vreinterpret_u8_u32(_src37ttt_b.val[1]);
  3596. _dst4.val[2] = vreinterpret_u8_u32(_src04ttt_b.val[0]);
  3597. _dst5.val[2] = vreinterpret_u8_u32(_src15ttt_b.val[0]);
  3598. _dst6.val[2] = vreinterpret_u8_u32(_src26ttt_b.val[0]);
  3599. _dst7.val[2] = vreinterpret_u8_u32(_src37ttt_b.val[0]);
  3600. vst3_u8(dst7, _dst7);
  3601. vst3_u8(dst6, _dst6);
  3602. vst3_u8(dst7 + dst_step, _dst5);
  3603. vst3_u8(dst6 + dst_step, _dst4);
  3604. vst3_u8(dst7 + 2 * dst_step, _dst3);
  3605. vst3_u8(dst6 + 2 * dst_step, _dst2);
  3606. vst3_u8(dst7 + 3 * dst_step, _dst1);
  3607. vst3_u8(dst6 + 3 * dst_step, _dst0);
  3608. src0 += 3 * 8;
  3609. src1 += 3 * 8;
  3610. dst7 += 4 * dst_step;
  3611. dst6 += 4 * dst_step;
  3612. }
  3613. #else
  3614. if (nn > 0)
  3615. {
  3616. asm volatile(
  3617. "0: \n"
  3618. "pld [%1, #192] \n"
  3619. "vld3.u8 {d0-d2}, [%1], %10 \n"
  3620. "pld [%2, #192] \n"
  3621. "vld3.u8 {d4-d6}, [%2], %10 \n"
  3622. "pld [%1, #192] \n"
  3623. "vld3.u8 {d8-d10}, [%1], %10 \n"
  3624. "vtrn.u8 q2, q0 \n" // _src01t_r
  3625. "vtrn.u8 d6, d2 \n"
  3626. "pld [%2, #192] \n"
  3627. "vld3.u8 {d12-d14}, [%2], %10\n"
  3628. "pld [%1, #192] \n"
  3629. "vld3.u8 {d16-d18}, [%1], %10\n"
  3630. "vtrn.u8 q6, q4 \n" // _src23t_r
  3631. "vtrn.u8 d14, d10 \n"
  3632. "pld [%2, #192] \n"
  3633. "vld3.u8 {d20-d22}, [%2], %10\n"
  3634. "pld [%1, #192] \n"
  3635. "vld3.u8 {d24-d26}, [%1], %10\n"
  3636. "vtrn.u8 q10, q8 \n" // _src45t_r
  3637. "vtrn.u8 d22, d18 \n"
  3638. "pld [%2, #192] \n"
  3639. "vld3.u8 {d28-d30}, [%2], %10\n"
  3640. "vtrn.u8 q14, q12 \n" // _src67t_r
  3641. "vtrn.u8 d30, d26 \n"
  3642. "sub %1, %1, %10, lsl #2 \n" // restore src0
  3643. "vtrn.u16 q4, q0 \n" // _src02tt_r
  3644. "vtrn.u16 d10, d2 \n"
  3645. "sub %2, %2, %10, lsl #2 \n" // restore src1
  3646. "vtrn.u16 q6, q2 \n" // _src13tt_r
  3647. "vtrn.u16 d14, d6 \n"
  3648. "add %1, #24 \n" // src0 += 24
  3649. "vtrn.u16 q12, q8 \n" // _src46tt_r
  3650. "vtrn.u16 d26, d18 \n"
  3651. "add %2, #24 \n" // src1 += 24
  3652. "vtrn.u16 q14, q10 \n" // _src57tt_r
  3653. "vtrn.u16 d30, d22 \n"
  3654. "vtrn.u32 q12, q4 \n" // _src26ttt_r
  3655. "vtrn.u32 d26, d10 \n"
  3656. "vtrn.u32 q14, q6 \n" // _src37ttt_r
  3657. "vst3.u8 {d24-d26}, [%4], %11\n"
  3658. "vtrn.u32 d30, d14 \n"
  3659. "vtrn.u32 q8, q0 \n" // _src04ttt_r
  3660. "vst3.u8 {d28-d30}, [%3], %11\n"
  3661. "vtrn.u32 d18, d2 \n"
  3662. "vtrn.u32 q10, q2 \n" // _src15ttt_r
  3663. "vst3.u8 {d16-d18}, [%4], %11\n"
  3664. "vtrn.u32 d22, d6 \n"
  3665. "subs %0, #1 \n"
  3666. "vst3.u8 {d8-d10}, [%4], %11 \n"
  3667. "vst3.u8 {d20-d22}, [%3], %11\n"
  3668. "vst3.u8 {d12-d14}, [%3], %11\n"
  3669. "vst3.u8 {d0-d2}, [%4], %11 \n"
  3670. "vst3.u8 {d4-d6}, [%3], %11 \n"
  3671. "bne 0b \n"
  3672. : "=r"(nn), // %0
  3673. "=r"(src0), // %1
  3674. "=r"(src1), // %2
  3675. "=r"(dst7), // %3
  3676. "=r"(dst6) // %4
  3677. : "0"(nn),
  3678. "1"(src0),
  3679. "2"(src1),
  3680. "3"(dst7),
  3681. "4"(dst6),
  3682. "r"(src_step), // %10
  3683. "r"(dst_step) // %11
  3684. : "cc", "memory", "q0", "q1", "q2", "q3", "q4", "q5", "q6", "q7", "q8", "q9", "q10", "q11", "q12", "q13", "q14", "q15");
  3685. }
  3686. #endif // __aarch64__
  3687. for (; remain > 0; remain--)
  3688. {
  3689. dst7[0] = src1[0 + 3 * src_step];
  3690. dst7[1] = src1[1 + 3 * src_step];
  3691. dst7[2] = src1[2 + 3 * src_step];
  3692. dst7[3] = src0[0 + 3 * src_step];
  3693. dst7[4] = src0[1 + 3 * src_step];
  3694. dst7[5] = src0[2 + 3 * src_step];
  3695. dst7[6] = src1[0 + 2 * src_step];
  3696. dst7[7] = src1[1 + 2 * src_step];
  3697. dst7[8] = src1[2 + 2 * src_step];
  3698. dst7[9] = src0[0 + 2 * src_step];
  3699. dst7[10] = src0[1 + 2 * src_step];
  3700. dst7[11] = src0[2 + 2 * src_step];
  3701. dst7[12] = src1[0 + src_step];
  3702. dst7[13] = src1[1 + src_step];
  3703. dst7[14] = src1[2 + src_step];
  3704. dst7[15] = src0[0 + src_step];
  3705. dst7[16] = src0[1 + src_step];
  3706. dst7[17] = src0[2 + src_step];
  3707. dst7[18] = src1[0];
  3708. dst7[19] = src1[1];
  3709. dst7[20] = src1[2];
  3710. dst7[21] = src0[0];
  3711. dst7[22] = src0[1];
  3712. dst7[23] = src0[2];
  3713. src0 += 3;
  3714. src1 += 3;
  3715. dst7 -= stride;
  3716. }
  3717. src0 += srcwgap + 7 * srcstride;
  3718. }
  3719. #endif // __ARM_NEON
  3720. for (; y < srch; y++)
  3721. {
  3722. unsigned char* dst0 = dstend - y * 3 - 3;
  3723. int x = 0;
  3724. for (; x < srcw; x++)
  3725. {
  3726. dst0[0] = src0[0];
  3727. dst0[1] = src0[1];
  3728. dst0[2] = src0[2];
  3729. src0 += 3;
  3730. dst0 -= stride;
  3731. }
  3732. src0 += srcwgap;
  3733. }
  3734. }
  3735. static void kanna_rotate_7_c4(const unsigned char* src, int srcw, int srch, int srcstride, unsigned char* dst, int w, int h, int stride)
  3736. {
  3737. const int srcwgap = srcstride - srcw * 4;
  3738. // point to the last dst pixel
  3739. unsigned char* dstend = dst + stride * (h - 1) + w * 4;
  3740. const unsigned char* src0 = src;
  3741. int y = 0;
  3742. #if __ARM_NEON
  3743. for (; y + 7 < srch; y += 8)
  3744. {
  3745. const unsigned char* src1 = src0 + srcstride;
  3746. unsigned char* dst6 = dstend - y * 4 - 8 * 4 - stride;
  3747. unsigned char* dst7 = dstend - y * 4 - 8 * 4;
  3748. int src_step = 2 * srcstride;
  3749. int dst_step = -2 * stride;
  3750. int nn = srcw >> 3;
  3751. int remain = srcw - (nn << 3);
  3752. #if !NCNN_GNU_INLINE_ASM || __aarch64__
  3753. for (; nn > 0; nn--)
  3754. {
  3755. uint8x8x4_t _src0 = vld4_u8(src0);
  3756. uint8x8x4_t _src1 = vld4_u8(src1);
  3757. uint8x8x4_t _src2 = vld4_u8(src0 + src_step);
  3758. uint8x8x4_t _src3 = vld4_u8(src1 + src_step);
  3759. uint8x8x4_t _src4 = vld4_u8(src0 + 2 * src_step);
  3760. uint8x8x4_t _src5 = vld4_u8(src1 + 2 * src_step);
  3761. uint8x8x4_t _src6 = vld4_u8(src0 + 3 * src_step);
  3762. uint8x8x4_t _src7 = vld4_u8(src1 + 3 * src_step);
  3763. uint8x8x2_t _src01t_r = vtrn_u8(_src1.val[0], _src0.val[0]);
  3764. uint8x8x2_t _src23t_r = vtrn_u8(_src3.val[0], _src2.val[0]);
  3765. uint8x8x2_t _src45t_r = vtrn_u8(_src5.val[0], _src4.val[0]);
  3766. uint8x8x2_t _src67t_r = vtrn_u8(_src7.val[0], _src6.val[0]);
  3767. uint8x8x2_t _src01t_g = vtrn_u8(_src1.val[1], _src0.val[1]);
  3768. uint8x8x2_t _src23t_g = vtrn_u8(_src3.val[1], _src2.val[1]);
  3769. uint8x8x2_t _src45t_g = vtrn_u8(_src5.val[1], _src4.val[1]);
  3770. uint8x8x2_t _src67t_g = vtrn_u8(_src7.val[1], _src6.val[1]);
  3771. uint8x8x2_t _src01t_b = vtrn_u8(_src1.val[2], _src0.val[2]);
  3772. uint8x8x2_t _src23t_b = vtrn_u8(_src3.val[2], _src2.val[2]);
  3773. uint8x8x2_t _src45t_b = vtrn_u8(_src5.val[2], _src4.val[2]);
  3774. uint8x8x2_t _src67t_b = vtrn_u8(_src7.val[2], _src6.val[2]);
  3775. uint8x8x2_t _src01t_a = vtrn_u8(_src1.val[3], _src0.val[3]);
  3776. uint8x8x2_t _src23t_a = vtrn_u8(_src3.val[3], _src2.val[3]);
  3777. uint8x8x2_t _src45t_a = vtrn_u8(_src5.val[3], _src4.val[3]);
  3778. uint8x8x2_t _src67t_a = vtrn_u8(_src7.val[3], _src6.val[3]);
  3779. uint16x4x2_t _src02tt_r = vtrn_u16(vreinterpret_u16_u8(_src23t_r.val[1]), vreinterpret_u16_u8(_src01t_r.val[1]));
  3780. uint16x4x2_t _src13tt_r = vtrn_u16(vreinterpret_u16_u8(_src23t_r.val[0]), vreinterpret_u16_u8(_src01t_r.val[0]));
  3781. uint16x4x2_t _src46tt_r = vtrn_u16(vreinterpret_u16_u8(_src67t_r.val[1]), vreinterpret_u16_u8(_src45t_r.val[1]));
  3782. uint16x4x2_t _src57tt_r = vtrn_u16(vreinterpret_u16_u8(_src67t_r.val[0]), vreinterpret_u16_u8(_src45t_r.val[0]));
  3783. uint16x4x2_t _src02tt_g = vtrn_u16(vreinterpret_u16_u8(_src23t_g.val[1]), vreinterpret_u16_u8(_src01t_g.val[1]));
  3784. uint16x4x2_t _src13tt_g = vtrn_u16(vreinterpret_u16_u8(_src23t_g.val[0]), vreinterpret_u16_u8(_src01t_g.val[0]));
  3785. uint16x4x2_t _src46tt_g = vtrn_u16(vreinterpret_u16_u8(_src67t_g.val[1]), vreinterpret_u16_u8(_src45t_g.val[1]));
  3786. uint16x4x2_t _src57tt_g = vtrn_u16(vreinterpret_u16_u8(_src67t_g.val[0]), vreinterpret_u16_u8(_src45t_g.val[0]));
  3787. uint16x4x2_t _src02tt_b = vtrn_u16(vreinterpret_u16_u8(_src23t_b.val[1]), vreinterpret_u16_u8(_src01t_b.val[1]));
  3788. uint16x4x2_t _src13tt_b = vtrn_u16(vreinterpret_u16_u8(_src23t_b.val[0]), vreinterpret_u16_u8(_src01t_b.val[0]));
  3789. uint16x4x2_t _src46tt_b = vtrn_u16(vreinterpret_u16_u8(_src67t_b.val[1]), vreinterpret_u16_u8(_src45t_b.val[1]));
  3790. uint16x4x2_t _src57tt_b = vtrn_u16(vreinterpret_u16_u8(_src67t_b.val[0]), vreinterpret_u16_u8(_src45t_b.val[0]));
  3791. uint16x4x2_t _src02tt_a = vtrn_u16(vreinterpret_u16_u8(_src23t_a.val[1]), vreinterpret_u16_u8(_src01t_a.val[1]));
  3792. uint16x4x2_t _src13tt_a = vtrn_u16(vreinterpret_u16_u8(_src23t_a.val[0]), vreinterpret_u16_u8(_src01t_a.val[0]));
  3793. uint16x4x2_t _src46tt_a = vtrn_u16(vreinterpret_u16_u8(_src67t_a.val[1]), vreinterpret_u16_u8(_src45t_a.val[1]));
  3794. uint16x4x2_t _src57tt_a = vtrn_u16(vreinterpret_u16_u8(_src67t_a.val[0]), vreinterpret_u16_u8(_src45t_a.val[0]));
  3795. uint32x2x2_t _src04ttt_r = vtrn_u32(vreinterpret_u32_u16(_src46tt_r.val[1]), vreinterpret_u32_u16(_src02tt_r.val[1]));
  3796. uint32x2x2_t _src15ttt_r = vtrn_u32(vreinterpret_u32_u16(_src57tt_r.val[1]), vreinterpret_u32_u16(_src13tt_r.val[1]));
  3797. uint32x2x2_t _src26ttt_r = vtrn_u32(vreinterpret_u32_u16(_src46tt_r.val[0]), vreinterpret_u32_u16(_src02tt_r.val[0]));
  3798. uint32x2x2_t _src37ttt_r = vtrn_u32(vreinterpret_u32_u16(_src57tt_r.val[0]), vreinterpret_u32_u16(_src13tt_r.val[0]));
  3799. uint32x2x2_t _src04ttt_g = vtrn_u32(vreinterpret_u32_u16(_src46tt_g.val[1]), vreinterpret_u32_u16(_src02tt_g.val[1]));
  3800. uint32x2x2_t _src15ttt_g = vtrn_u32(vreinterpret_u32_u16(_src57tt_g.val[1]), vreinterpret_u32_u16(_src13tt_g.val[1]));
  3801. uint32x2x2_t _src26ttt_g = vtrn_u32(vreinterpret_u32_u16(_src46tt_g.val[0]), vreinterpret_u32_u16(_src02tt_g.val[0]));
  3802. uint32x2x2_t _src37ttt_g = vtrn_u32(vreinterpret_u32_u16(_src57tt_g.val[0]), vreinterpret_u32_u16(_src13tt_g.val[0]));
  3803. uint32x2x2_t _src04ttt_b = vtrn_u32(vreinterpret_u32_u16(_src46tt_b.val[1]), vreinterpret_u32_u16(_src02tt_b.val[1]));
  3804. uint32x2x2_t _src15ttt_b = vtrn_u32(vreinterpret_u32_u16(_src57tt_b.val[1]), vreinterpret_u32_u16(_src13tt_b.val[1]));
  3805. uint32x2x2_t _src26ttt_b = vtrn_u32(vreinterpret_u32_u16(_src46tt_b.val[0]), vreinterpret_u32_u16(_src02tt_b.val[0]));
  3806. uint32x2x2_t _src37ttt_b = vtrn_u32(vreinterpret_u32_u16(_src57tt_b.val[0]), vreinterpret_u32_u16(_src13tt_b.val[0]));
  3807. uint32x2x2_t _src04ttt_a = vtrn_u32(vreinterpret_u32_u16(_src46tt_a.val[1]), vreinterpret_u32_u16(_src02tt_a.val[1]));
  3808. uint32x2x2_t _src15ttt_a = vtrn_u32(vreinterpret_u32_u16(_src57tt_a.val[1]), vreinterpret_u32_u16(_src13tt_a.val[1]));
  3809. uint32x2x2_t _src26ttt_a = vtrn_u32(vreinterpret_u32_u16(_src46tt_a.val[0]), vreinterpret_u32_u16(_src02tt_a.val[0]));
  3810. uint32x2x2_t _src37ttt_a = vtrn_u32(vreinterpret_u32_u16(_src57tt_a.val[0]), vreinterpret_u32_u16(_src13tt_a.val[0]));
  3811. uint8x8x4_t _dst0;
  3812. uint8x8x4_t _dst1;
  3813. uint8x8x4_t _dst2;
  3814. uint8x8x4_t _dst3;
  3815. uint8x8x4_t _dst4;
  3816. uint8x8x4_t _dst5;
  3817. uint8x8x4_t _dst6;
  3818. uint8x8x4_t _dst7;
  3819. _dst0.val[0] = vreinterpret_u8_u32(_src04ttt_r.val[1]);
  3820. _dst1.val[0] = vreinterpret_u8_u32(_src15ttt_r.val[1]);
  3821. _dst2.val[0] = vreinterpret_u8_u32(_src26ttt_r.val[1]);
  3822. _dst3.val[0] = vreinterpret_u8_u32(_src37ttt_r.val[1]);
  3823. _dst4.val[0] = vreinterpret_u8_u32(_src04ttt_r.val[0]);
  3824. _dst5.val[0] = vreinterpret_u8_u32(_src15ttt_r.val[0]);
  3825. _dst6.val[0] = vreinterpret_u8_u32(_src26ttt_r.val[0]);
  3826. _dst7.val[0] = vreinterpret_u8_u32(_src37ttt_r.val[0]);
  3827. _dst0.val[1] = vreinterpret_u8_u32(_src04ttt_g.val[1]);
  3828. _dst1.val[1] = vreinterpret_u8_u32(_src15ttt_g.val[1]);
  3829. _dst2.val[1] = vreinterpret_u8_u32(_src26ttt_g.val[1]);
  3830. _dst3.val[1] = vreinterpret_u8_u32(_src37ttt_g.val[1]);
  3831. _dst4.val[1] = vreinterpret_u8_u32(_src04ttt_g.val[0]);
  3832. _dst5.val[1] = vreinterpret_u8_u32(_src15ttt_g.val[0]);
  3833. _dst6.val[1] = vreinterpret_u8_u32(_src26ttt_g.val[0]);
  3834. _dst7.val[1] = vreinterpret_u8_u32(_src37ttt_g.val[0]);
  3835. _dst0.val[2] = vreinterpret_u8_u32(_src04ttt_b.val[1]);
  3836. _dst1.val[2] = vreinterpret_u8_u32(_src15ttt_b.val[1]);
  3837. _dst2.val[2] = vreinterpret_u8_u32(_src26ttt_b.val[1]);
  3838. _dst3.val[2] = vreinterpret_u8_u32(_src37ttt_b.val[1]);
  3839. _dst4.val[2] = vreinterpret_u8_u32(_src04ttt_b.val[0]);
  3840. _dst5.val[2] = vreinterpret_u8_u32(_src15ttt_b.val[0]);
  3841. _dst6.val[2] = vreinterpret_u8_u32(_src26ttt_b.val[0]);
  3842. _dst7.val[2] = vreinterpret_u8_u32(_src37ttt_b.val[0]);
  3843. _dst0.val[3] = vreinterpret_u8_u32(_src04ttt_a.val[1]);
  3844. _dst1.val[3] = vreinterpret_u8_u32(_src15ttt_a.val[1]);
  3845. _dst2.val[3] = vreinterpret_u8_u32(_src26ttt_a.val[1]);
  3846. _dst3.val[3] = vreinterpret_u8_u32(_src37ttt_a.val[1]);
  3847. _dst4.val[3] = vreinterpret_u8_u32(_src04ttt_a.val[0]);
  3848. _dst5.val[3] = vreinterpret_u8_u32(_src15ttt_a.val[0]);
  3849. _dst6.val[3] = vreinterpret_u8_u32(_src26ttt_a.val[0]);
  3850. _dst7.val[3] = vreinterpret_u8_u32(_src37ttt_a.val[0]);
  3851. vst4_u8(dst7, _dst7);
  3852. vst4_u8(dst6, _dst6);
  3853. vst4_u8(dst7 + dst_step, _dst5);
  3854. vst4_u8(dst6 + dst_step, _dst4);
  3855. vst4_u8(dst7 + 2 * dst_step, _dst3);
  3856. vst4_u8(dst6 + 2 * dst_step, _dst2);
  3857. vst4_u8(dst7 + 3 * dst_step, _dst1);
  3858. vst4_u8(dst6 + 3 * dst_step, _dst0);
  3859. src0 += 4 * 8;
  3860. src1 += 4 * 8;
  3861. dst7 += 4 * dst_step;
  3862. dst6 += 4 * dst_step;
  3863. }
  3864. #else
  3865. if (nn > 0)
  3866. {
  3867. asm volatile(
  3868. "0: \n"
  3869. "pld [%1, #256] \n"
  3870. "vld4.u8 {d0-d3}, [%1], %10 \n"
  3871. "pld [%2, #256] \n"
  3872. "vld4.u8 {d4-d7}, [%2], %10 \n"
  3873. "pld [%1, #256] \n"
  3874. "vld4.u8 {d8-d11}, [%1], %10 \n"
  3875. "vtrn.u8 q2, q0 \n" // _src01t_r
  3876. "vtrn.u8 q3, q1 \n"
  3877. "pld [%2, #256] \n"
  3878. "vld4.u8 {d12-d15}, [%2], %10\n"
  3879. "pld [%1, #256] \n"
  3880. "vld4.u8 {d16-d19}, [%1], %10\n"
  3881. "vtrn.u8 q6, q4 \n" // _src23t_r
  3882. "vtrn.u8 q7, q5 \n"
  3883. "pld [%2, #256] \n"
  3884. "vld4.u8 {d20-d23}, [%2], %10\n"
  3885. "pld [%1, #256] \n"
  3886. "vld4.u8 {d24-d27}, [%1], %10\n"
  3887. "vtrn.u8 q10, q8 \n" // _src45t_r
  3888. "vtrn.u8 q11, q9 \n"
  3889. "pld [%2, #256] \n"
  3890. "vld4.u8 {d28-d31}, [%2], %10\n"
  3891. "vtrn.u8 q14, q12 \n" // _src67t_r
  3892. "vtrn.u8 q15, q13 \n"
  3893. "sub %1, %1, %10, lsl #2 \n" // restore src0
  3894. "vtrn.u16 q4, q0 \n" // _src02tt_r
  3895. "vtrn.u16 q5, q1 \n"
  3896. "sub %2, %2, %10, lsl #2 \n" // restore src1
  3897. "vtrn.u16 q6, q2 \n" // _src13tt_r
  3898. "vtrn.u16 q7, q3 \n"
  3899. "add %1, #32 \n" // src0 += 32
  3900. "vtrn.u16 q12, q8 \n" // _src46tt_r
  3901. "vtrn.u16 q13, q9 \n"
  3902. "add %2, #32 \n" // src1 += 32
  3903. "vtrn.u16 q14, q10 \n" // _src57tt_r
  3904. "vtrn.u16 q15, q11 \n"
  3905. "vtrn.u32 q12, q4 \n" // _src26ttt_r
  3906. "vtrn.u32 q13, q5 \n"
  3907. "vtrn.u32 q14, q6 \n" // _src37ttt_r
  3908. "vst4.u8 {d24-d27}, [%4], %11\n"
  3909. "vtrn.u32 q15, q7 \n"
  3910. "vtrn.u32 q8, q0 \n" // _src04ttt_r
  3911. "vst4.u8 {d28-d31}, [%3], %11\n"
  3912. "vtrn.u32 q9, q1 \n"
  3913. "vtrn.u32 q10, q2 \n" // _src15ttt_r
  3914. "vst4.u8 {d16-d19}, [%4], %11\n"
  3915. "vtrn.u32 q11, q3 \n"
  3916. "subs %0, #1 \n"
  3917. "vst4.u8 {d8-d11}, [%4], %11 \n"
  3918. "vst4.u8 {d20-d23}, [%3], %11\n"
  3919. "vst4.u8 {d12-d15}, [%3], %11\n"
  3920. "vst4.u8 {d0-d3}, [%4], %11 \n"
  3921. "vst4.u8 {d4-d7}, [%3], %11 \n"
  3922. "bne 0b \n"
  3923. : "=r"(nn), // %0
  3924. "=r"(src0), // %1
  3925. "=r"(src1), // %2
  3926. "=r"(dst7), // %3
  3927. "=r"(dst6) // %4
  3928. : "0"(nn),
  3929. "1"(src0),
  3930. "2"(src1),
  3931. "3"(dst7),
  3932. "4"(dst6),
  3933. "r"(src_step), // %10
  3934. "r"(dst_step) // %11
  3935. : "cc", "memory", "q0", "q1", "q2", "q3", "q4", "q5", "q6", "q7", "q8", "q9", "q10", "q11", "q12", "q13", "q14", "q15");
  3936. }
  3937. #endif // __aarch64__
  3938. for (; remain > 0; remain--)
  3939. {
  3940. dst7[0] = src1[0 + 3 * src_step];
  3941. dst7[1] = src1[1 + 3 * src_step];
  3942. dst7[2] = src1[2 + 3 * src_step];
  3943. dst7[3] = src1[3 + 3 * src_step];
  3944. dst7[4] = src0[0 + 3 * src_step];
  3945. dst7[5] = src0[1 + 3 * src_step];
  3946. dst7[6] = src0[2 + 3 * src_step];
  3947. dst7[7] = src0[3 + 3 * src_step];
  3948. dst7[8] = src1[0 + 2 * src_step];
  3949. dst7[9] = src1[1 + 2 * src_step];
  3950. dst7[10] = src1[2 + 2 * src_step];
  3951. dst7[11] = src1[3 + 2 * src_step];
  3952. dst7[12] = src0[0 + 2 * src_step];
  3953. dst7[13] = src0[1 + 2 * src_step];
  3954. dst7[14] = src0[2 + 2 * src_step];
  3955. dst7[15] = src0[3 + 2 * src_step];
  3956. dst7[16] = src1[0 + src_step];
  3957. dst7[17] = src1[1 + src_step];
  3958. dst7[18] = src1[2 + src_step];
  3959. dst7[19] = src1[3 + src_step];
  3960. dst7[20] = src0[0 + src_step];
  3961. dst7[21] = src0[1 + src_step];
  3962. dst7[22] = src0[2 + src_step];
  3963. dst7[23] = src0[3 + src_step];
  3964. dst7[24] = src1[0];
  3965. dst7[25] = src1[1];
  3966. dst7[26] = src1[2];
  3967. dst7[27] = src1[3];
  3968. dst7[28] = src0[0];
  3969. dst7[29] = src0[1];
  3970. dst7[30] = src0[2];
  3971. dst7[31] = src0[3];
  3972. src0 += 4;
  3973. src1 += 4;
  3974. dst7 -= stride;
  3975. }
  3976. src0 += srcwgap + 7 * srcstride;
  3977. }
  3978. #endif // __ARM_NEON
  3979. for (; y < srch; y++)
  3980. {
  3981. unsigned char* dst0 = dstend - y * 4 - 4;
  3982. int x = 0;
  3983. for (; x < srcw; x++)
  3984. {
  3985. dst0[0] = src0[0];
  3986. dst0[1] = src0[1];
  3987. dst0[2] = src0[2];
  3988. dst0[3] = src0[3];
  3989. src0 += 4;
  3990. dst0 -= stride;
  3991. }
  3992. src0 += srcwgap;
  3993. }
  3994. }
  3995. static void kanna_rotate_8_c1(const unsigned char* src, int srcw, int srch, int srcstride, unsigned char* dst, int /*w*/, int h, int stride)
  3996. {
  3997. const int srcwgap = srcstride - srcw;
  3998. // point to the last dst pixel row
  3999. unsigned char* dstend = dst + stride * (h - 1);
  4000. const unsigned char* src0 = src;
  4001. int y = 0;
  4002. #if __ARM_NEON
  4003. for (; y + 7 < srch; y += 8)
  4004. {
  4005. const unsigned char* src1 = src0 + srcstride;
  4006. unsigned char* dst7 = dstend + y;
  4007. unsigned char* dst6 = dstend + y - stride;
  4008. int src_step = 2 * srcstride;
  4009. int dst_step = -2 * stride;
  4010. int nn = srcw >> 3;
  4011. int remain = srcw - (nn << 3);
  4012. #if !NCNN_GNU_INLINE_ASM || __aarch64__
  4013. for (; nn > 0; nn--)
  4014. {
  4015. uint8x8_t _src0 = vld1_u8(src0);
  4016. uint8x8_t _src1 = vld1_u8(src1);
  4017. uint8x8_t _src2 = vld1_u8(src0 + src_step);
  4018. uint8x8_t _src3 = vld1_u8(src1 + src_step);
  4019. uint8x8_t _src4 = vld1_u8(src0 + 2 * src_step);
  4020. uint8x8_t _src5 = vld1_u8(src1 + 2 * src_step);
  4021. uint8x8_t _src6 = vld1_u8(src0 + 3 * src_step);
  4022. uint8x8_t _src7 = vld1_u8(src1 + 3 * src_step);
  4023. uint8x8x2_t _src01t_r = vtrn_u8(_src0, _src1);
  4024. uint8x8x2_t _src23t_r = vtrn_u8(_src2, _src3);
  4025. uint8x8x2_t _src45t_r = vtrn_u8(_src4, _src5);
  4026. uint8x8x2_t _src67t_r = vtrn_u8(_src6, _src7);
  4027. uint16x4x2_t _src02tt_r = vtrn_u16(vreinterpret_u16_u8(_src01t_r.val[0]), vreinterpret_u16_u8(_src23t_r.val[0]));
  4028. uint16x4x2_t _src13tt_r = vtrn_u16(vreinterpret_u16_u8(_src01t_r.val[1]), vreinterpret_u16_u8(_src23t_r.val[1]));
  4029. uint16x4x2_t _src46tt_r = vtrn_u16(vreinterpret_u16_u8(_src45t_r.val[0]), vreinterpret_u16_u8(_src67t_r.val[0]));
  4030. uint16x4x2_t _src57tt_r = vtrn_u16(vreinterpret_u16_u8(_src45t_r.val[1]), vreinterpret_u16_u8(_src67t_r.val[1]));
  4031. uint32x2x2_t _src04ttt_r = vtrn_u32(vreinterpret_u32_u16(_src02tt_r.val[0]), vreinterpret_u32_u16(_src46tt_r.val[0]));
  4032. uint32x2x2_t _src15ttt_r = vtrn_u32(vreinterpret_u32_u16(_src13tt_r.val[0]), vreinterpret_u32_u16(_src57tt_r.val[0]));
  4033. uint32x2x2_t _src26ttt_r = vtrn_u32(vreinterpret_u32_u16(_src02tt_r.val[1]), vreinterpret_u32_u16(_src46tt_r.val[1]));
  4034. uint32x2x2_t _src37ttt_r = vtrn_u32(vreinterpret_u32_u16(_src13tt_r.val[1]), vreinterpret_u32_u16(_src57tt_r.val[1]));
  4035. uint8x8_t _dst0 = vreinterpret_u8_u32(_src04ttt_r.val[0]);
  4036. uint8x8_t _dst1 = vreinterpret_u8_u32(_src15ttt_r.val[0]);
  4037. uint8x8_t _dst2 = vreinterpret_u8_u32(_src26ttt_r.val[0]);
  4038. uint8x8_t _dst3 = vreinterpret_u8_u32(_src37ttt_r.val[0]);
  4039. uint8x8_t _dst4 = vreinterpret_u8_u32(_src04ttt_r.val[1]);
  4040. uint8x8_t _dst5 = vreinterpret_u8_u32(_src15ttt_r.val[1]);
  4041. uint8x8_t _dst6 = vreinterpret_u8_u32(_src26ttt_r.val[1]);
  4042. uint8x8_t _dst7 = vreinterpret_u8_u32(_src37ttt_r.val[1]);
  4043. vst1_u8(dst7, _dst0);
  4044. vst1_u8(dst6, _dst1);
  4045. vst1_u8(dst7 + dst_step, _dst2);
  4046. vst1_u8(dst6 + dst_step, _dst3);
  4047. vst1_u8(dst7 + 2 * dst_step, _dst4);
  4048. vst1_u8(dst6 + 2 * dst_step, _dst5);
  4049. vst1_u8(dst7 + 3 * dst_step, _dst6);
  4050. vst1_u8(dst6 + 3 * dst_step, _dst7);
  4051. src0 += 8;
  4052. src1 += 8;
  4053. dst7 += 4 * dst_step;
  4054. dst6 += 4 * dst_step;
  4055. }
  4056. #else
  4057. if (nn > 0)
  4058. {
  4059. asm volatile(
  4060. "0: \n"
  4061. "pld [%1, #64] \n"
  4062. "vld1.u8 {d0}, [%1], %10 \n"
  4063. "pld [%2, #64] \n"
  4064. "vld1.u8 {d1}, [%2], %10 \n"
  4065. "pld [%1, #64] \n"
  4066. "vld1.u8 {d2}, [%1], %10 \n"
  4067. "vtrn.u8 d0, d1 \n" // _src01t_r
  4068. "pld [%2, #64] \n"
  4069. "vld1.u8 {d3}, [%2], %10 \n"
  4070. "pld [%1, #64] \n"
  4071. "vld1.u8 {d4}, [%1], %10 \n"
  4072. "vtrn.u8 d2, d3 \n" // _src23t_r
  4073. "pld [%2, #64] \n"
  4074. "vld1.u8 {d5}, [%2], %10 \n"
  4075. "pld [%1, #64] \n"
  4076. "vld1.u8 {d6}, [%1], %10 \n"
  4077. "vtrn.u8 d4, d5 \n" // _src45t_r
  4078. "pld [%2, #64] \n"
  4079. "vld1.u8 {d7}, [%2], %10 \n"
  4080. "vtrn.u8 d6, d7 \n" // _src67t_r
  4081. "sub %1, %1, %10, lsl #2 \n" // restore src0
  4082. "vtrn.u16 q0, q1 \n" // _src02tt_r _src13tt_r
  4083. "sub %2, %2, %10, lsl #2 \n" // restore src1
  4084. "vtrn.u16 q2, q3 \n" // _src46tt_r _src57tt_r
  4085. "add %1, #8 \n" // src0 += 8
  4086. "vtrn.u32 q0, q2 \n" // _src04ttt_r _src15ttt_r
  4087. "add %2, #8 \n" // src1 += 8
  4088. "vtrn.u32 q1, q3 \n" // _src26ttt_r _src37ttt_r
  4089. "vst1.u8 {d0}, [%3], %11 \n"
  4090. "vst1.u8 {d1}, [%4], %11 \n"
  4091. "subs %0, #1 \n"
  4092. "vst1.u8 {d2}, [%3], %11 \n"
  4093. "vst1.u8 {d3}, [%4], %11 \n"
  4094. "vst1.u8 {d4}, [%3], %11 \n"
  4095. "vst1.u8 {d5}, [%4], %11 \n"
  4096. "vst1.u8 {d6}, [%3], %11 \n"
  4097. "vst1.u8 {d7}, [%4], %11 \n"
  4098. "bne 0b \n"
  4099. : "=r"(nn), // %0
  4100. "=r"(src0), // %1
  4101. "=r"(src1), // %2
  4102. "=r"(dst7), // %3
  4103. "=r"(dst6) // %4
  4104. : "0"(nn),
  4105. "1"(src0),
  4106. "2"(src1),
  4107. "3"(dst7),
  4108. "4"(dst6),
  4109. "r"(src_step), // %10
  4110. "r"(dst_step) // %11
  4111. : "cc", "memory", "q0", "q1", "q2", "q3");
  4112. }
  4113. #endif // __aarch64__
  4114. for (; remain > 0; remain--)
  4115. {
  4116. dst7[0] = src0[0];
  4117. dst7[1] = src1[0];
  4118. dst7[2] = src0[0 + src_step];
  4119. dst7[3] = src1[0 + src_step];
  4120. dst7[4] = src0[0 + 2 * src_step];
  4121. dst7[5] = src1[0 + 2 * src_step];
  4122. dst7[6] = src0[0 + 3 * src_step];
  4123. dst7[7] = src1[0 + 3 * src_step];
  4124. src0 += 1;
  4125. src1 += 1;
  4126. dst7 -= stride;
  4127. }
  4128. src0 += srcwgap + 7 * srcstride;
  4129. }
  4130. #endif // __ARM_NEON
  4131. for (; y < srch; y++)
  4132. {
  4133. unsigned char* dst0 = dstend + y;
  4134. int x = 0;
  4135. for (; x < srcw; x++)
  4136. {
  4137. *dst0 = *src0;
  4138. src0 += 1;
  4139. dst0 -= stride;
  4140. }
  4141. src0 += srcwgap;
  4142. }
  4143. }
  4144. static void kanna_rotate_8_c2(const unsigned char* src, int srcw, int srch, int srcstride, unsigned char* dst, int /*w*/, int h, int stride)
  4145. {
  4146. const int srcwgap = srcstride - srcw * 2;
  4147. // point to the last dst pixel row
  4148. unsigned char* dstend = dst + stride * (h - 1);
  4149. const unsigned char* src0 = src;
  4150. int y = 0;
  4151. #if __ARM_NEON
  4152. for (; y + 7 < srch; y += 8)
  4153. {
  4154. const unsigned char* src1 = src0 + srcstride;
  4155. unsigned char* dst7 = dstend + y * 2;
  4156. unsigned char* dst6 = dstend + y * 2 - stride;
  4157. int src_step = 2 * srcstride;
  4158. int dst_step = -2 * stride;
  4159. int nn = srcw >> 3;
  4160. int remain = srcw - (nn << 3);
  4161. #if !NCNN_GNU_INLINE_ASM || __aarch64__
  4162. for (; nn > 0; nn--)
  4163. {
  4164. uint8x8x2_t _src0 = vld2_u8(src0);
  4165. uint8x8x2_t _src1 = vld2_u8(src1);
  4166. uint8x8x2_t _src2 = vld2_u8(src0 + src_step);
  4167. uint8x8x2_t _src3 = vld2_u8(src1 + src_step);
  4168. uint8x8x2_t _src4 = vld2_u8(src0 + 2 * src_step);
  4169. uint8x8x2_t _src5 = vld2_u8(src1 + 2 * src_step);
  4170. uint8x8x2_t _src6 = vld2_u8(src0 + 3 * src_step);
  4171. uint8x8x2_t _src7 = vld2_u8(src1 + 3 * src_step);
  4172. uint8x8x2_t _src01t_r = vtrn_u8(_src0.val[0], _src1.val[0]);
  4173. uint8x8x2_t _src23t_r = vtrn_u8(_src2.val[0], _src3.val[0]);
  4174. uint8x8x2_t _src45t_r = vtrn_u8(_src4.val[0], _src5.val[0]);
  4175. uint8x8x2_t _src67t_r = vtrn_u8(_src6.val[0], _src7.val[0]);
  4176. uint8x8x2_t _src01t_g = vtrn_u8(_src0.val[1], _src1.val[1]);
  4177. uint8x8x2_t _src23t_g = vtrn_u8(_src2.val[1], _src3.val[1]);
  4178. uint8x8x2_t _src45t_g = vtrn_u8(_src4.val[1], _src5.val[1]);
  4179. uint8x8x2_t _src67t_g = vtrn_u8(_src6.val[1], _src7.val[1]);
  4180. uint16x4x2_t _src02tt_r = vtrn_u16(vreinterpret_u16_u8(_src01t_r.val[0]), vreinterpret_u16_u8(_src23t_r.val[0]));
  4181. uint16x4x2_t _src13tt_r = vtrn_u16(vreinterpret_u16_u8(_src01t_r.val[1]), vreinterpret_u16_u8(_src23t_r.val[1]));
  4182. uint16x4x2_t _src46tt_r = vtrn_u16(vreinterpret_u16_u8(_src45t_r.val[0]), vreinterpret_u16_u8(_src67t_r.val[0]));
  4183. uint16x4x2_t _src57tt_r = vtrn_u16(vreinterpret_u16_u8(_src45t_r.val[1]), vreinterpret_u16_u8(_src67t_r.val[1]));
  4184. uint16x4x2_t _src02tt_g = vtrn_u16(vreinterpret_u16_u8(_src01t_g.val[0]), vreinterpret_u16_u8(_src23t_g.val[0]));
  4185. uint16x4x2_t _src13tt_g = vtrn_u16(vreinterpret_u16_u8(_src01t_g.val[1]), vreinterpret_u16_u8(_src23t_g.val[1]));
  4186. uint16x4x2_t _src46tt_g = vtrn_u16(vreinterpret_u16_u8(_src45t_g.val[0]), vreinterpret_u16_u8(_src67t_g.val[0]));
  4187. uint16x4x2_t _src57tt_g = vtrn_u16(vreinterpret_u16_u8(_src45t_g.val[1]), vreinterpret_u16_u8(_src67t_g.val[1]));
  4188. uint32x2x2_t _src04ttt_r = vtrn_u32(vreinterpret_u32_u16(_src02tt_r.val[0]), vreinterpret_u32_u16(_src46tt_r.val[0]));
  4189. uint32x2x2_t _src15ttt_r = vtrn_u32(vreinterpret_u32_u16(_src13tt_r.val[0]), vreinterpret_u32_u16(_src57tt_r.val[0]));
  4190. uint32x2x2_t _src26ttt_r = vtrn_u32(vreinterpret_u32_u16(_src02tt_r.val[1]), vreinterpret_u32_u16(_src46tt_r.val[1]));
  4191. uint32x2x2_t _src37ttt_r = vtrn_u32(vreinterpret_u32_u16(_src13tt_r.val[1]), vreinterpret_u32_u16(_src57tt_r.val[1]));
  4192. uint32x2x2_t _src04ttt_g = vtrn_u32(vreinterpret_u32_u16(_src02tt_g.val[0]), vreinterpret_u32_u16(_src46tt_g.val[0]));
  4193. uint32x2x2_t _src15ttt_g = vtrn_u32(vreinterpret_u32_u16(_src13tt_g.val[0]), vreinterpret_u32_u16(_src57tt_g.val[0]));
  4194. uint32x2x2_t _src26ttt_g = vtrn_u32(vreinterpret_u32_u16(_src02tt_g.val[1]), vreinterpret_u32_u16(_src46tt_g.val[1]));
  4195. uint32x2x2_t _src37ttt_g = vtrn_u32(vreinterpret_u32_u16(_src13tt_g.val[1]), vreinterpret_u32_u16(_src57tt_g.val[1]));
  4196. uint8x8x2_t _dst0;
  4197. uint8x8x2_t _dst1;
  4198. uint8x8x2_t _dst2;
  4199. uint8x8x2_t _dst3;
  4200. uint8x8x2_t _dst4;
  4201. uint8x8x2_t _dst5;
  4202. uint8x8x2_t _dst6;
  4203. uint8x8x2_t _dst7;
  4204. _dst0.val[0] = vreinterpret_u8_u32(_src04ttt_r.val[0]);
  4205. _dst1.val[0] = vreinterpret_u8_u32(_src15ttt_r.val[0]);
  4206. _dst2.val[0] = vreinterpret_u8_u32(_src26ttt_r.val[0]);
  4207. _dst3.val[0] = vreinterpret_u8_u32(_src37ttt_r.val[0]);
  4208. _dst4.val[0] = vreinterpret_u8_u32(_src04ttt_r.val[1]);
  4209. _dst5.val[0] = vreinterpret_u8_u32(_src15ttt_r.val[1]);
  4210. _dst6.val[0] = vreinterpret_u8_u32(_src26ttt_r.val[1]);
  4211. _dst7.val[0] = vreinterpret_u8_u32(_src37ttt_r.val[1]);
  4212. _dst0.val[1] = vreinterpret_u8_u32(_src04ttt_g.val[0]);
  4213. _dst1.val[1] = vreinterpret_u8_u32(_src15ttt_g.val[0]);
  4214. _dst2.val[1] = vreinterpret_u8_u32(_src26ttt_g.val[0]);
  4215. _dst3.val[1] = vreinterpret_u8_u32(_src37ttt_g.val[0]);
  4216. _dst4.val[1] = vreinterpret_u8_u32(_src04ttt_g.val[1]);
  4217. _dst5.val[1] = vreinterpret_u8_u32(_src15ttt_g.val[1]);
  4218. _dst6.val[1] = vreinterpret_u8_u32(_src26ttt_g.val[1]);
  4219. _dst7.val[1] = vreinterpret_u8_u32(_src37ttt_g.val[1]);
  4220. vst2_u8(dst7, _dst0);
  4221. vst2_u8(dst6, _dst1);
  4222. vst2_u8(dst7 + dst_step, _dst2);
  4223. vst2_u8(dst6 + dst_step, _dst3);
  4224. vst2_u8(dst7 + 2 * dst_step, _dst4);
  4225. vst2_u8(dst6 + 2 * dst_step, _dst5);
  4226. vst2_u8(dst7 + 3 * dst_step, _dst6);
  4227. vst2_u8(dst6 + 3 * dst_step, _dst7);
  4228. src0 += 2 * 8;
  4229. src1 += 2 * 8;
  4230. dst7 += 4 * dst_step;
  4231. dst6 += 4 * dst_step;
  4232. }
  4233. #else
  4234. if (nn > 0)
  4235. {
  4236. asm volatile(
  4237. "0: \n"
  4238. "pld [%1, #128] \n"
  4239. "vld2.u8 {d0-d1}, [%1], %10 \n"
  4240. "pld [%2, #128] \n"
  4241. "vld2.u8 {d2-d3}, [%2], %10 \n"
  4242. "pld [%1, #128] \n"
  4243. "vld2.u8 {d4-d5}, [%1], %10 \n"
  4244. "vtrn.u8 q0, q1 \n" // _src01t_r
  4245. "pld [%2, #128] \n"
  4246. "vld2.u8 {d6-d7}, [%2], %10 \n"
  4247. "pld [%1, #128] \n"
  4248. "vld2.u8 {d16-d17}, [%1], %10\n"
  4249. "vtrn.u8 q2, q3 \n" // _src23t_r
  4250. "pld [%2, #128] \n"
  4251. "vld2.u8 {d18-d19}, [%2], %10\n"
  4252. "pld [%1, #128] \n"
  4253. "vld2.u8 {d20-d21}, [%1], %10\n"
  4254. "vtrn.u8 q8, q9 \n" // _src45t_r
  4255. "pld [%2, #128] \n"
  4256. "vld2.u8 {d22-d23}, [%2], %10\n"
  4257. "vtrn.u8 q10, q11 \n" // _src67t_r
  4258. "sub %1, %1, %10, lsl #2 \n" // restore src0
  4259. "vtrn.u16 q0, q2 \n" // _src02tt_r
  4260. "sub %2, %2, %10, lsl #2 \n" // restore src1
  4261. "vtrn.u16 q1, q3 \n" // _src13tt_r
  4262. "add %1, #16 \n" // src0 += 16
  4263. "vtrn.u16 q8, q10 \n" // _src46tt_r
  4264. "add %2, #16 \n" // src1 += 16
  4265. "vtrn.u16 q9, q11 \n" // _src57tt_r
  4266. "vtrn.u32 q0, q8 \n" // _src04ttt_r
  4267. "vtrn.u32 q1, q9 \n" // _src15ttt_r
  4268. "vst2.u8 {d0-d1}, [%3], %11 \n"
  4269. "vtrn.u32 q2, q10 \n" // _src26ttt_r
  4270. "vst2.u8 {d2-d3}, [%4], %11 \n"
  4271. "vtrn.u32 q3, q11 \n" // _src37ttt_r
  4272. "vst2.u8 {d4-d5}, [%3], %11 \n"
  4273. "subs %0, #1 \n"
  4274. "vst2.u8 {d16-d17}, [%3], %11\n"
  4275. "vst2.u8 {d6-d7}, [%4], %11 \n"
  4276. "vst2.u8 {d18-d19}, [%4], %11\n"
  4277. "vst2.u8 {d20-d21}, [%3], %11\n"
  4278. "vst2.u8 {d22-d23}, [%4], %11\n"
  4279. "bne 0b \n"
  4280. : "=r"(nn), // %0
  4281. "=r"(src0), // %1
  4282. "=r"(src1), // %2
  4283. "=r"(dst7), // %3
  4284. "=r"(dst6) // %4
  4285. : "0"(nn),
  4286. "1"(src0),
  4287. "2"(src1),
  4288. "3"(dst7),
  4289. "4"(dst6),
  4290. "r"(src_step), // %10
  4291. "r"(dst_step) // %11
  4292. : "cc", "memory", "q0", "q1", "q2", "q3", "q8", "q9", "q10", "q11");
  4293. }
  4294. #endif // __aarch64__
  4295. for (; remain > 0; remain--)
  4296. {
  4297. dst7[0] = src0[0];
  4298. dst7[1] = src0[1];
  4299. dst7[2] = src1[0];
  4300. dst7[3] = src1[1];
  4301. dst7[4] = src0[0 + src_step];
  4302. dst7[5] = src0[1 + src_step];
  4303. dst7[6] = src1[0 + src_step];
  4304. dst7[7] = src1[1 + src_step];
  4305. dst7[8] = src0[0 + 2 * src_step];
  4306. dst7[9] = src0[1 + 2 * src_step];
  4307. dst7[10] = src1[0 + 2 * src_step];
  4308. dst7[11] = src1[1 + 2 * src_step];
  4309. dst7[12] = src0[0 + 3 * src_step];
  4310. dst7[13] = src0[1 + 3 * src_step];
  4311. dst7[14] = src1[0 + 3 * src_step];
  4312. dst7[15] = src1[1 + 3 * src_step];
  4313. src0 += 2;
  4314. src1 += 2;
  4315. dst7 -= stride;
  4316. }
  4317. src0 += srcwgap + 7 * srcstride;
  4318. }
  4319. #endif // __ARM_NEON
  4320. for (; y < srch; y++)
  4321. {
  4322. unsigned char* dst0 = dstend + y * 2;
  4323. int x = 0;
  4324. for (; x < srcw; x++)
  4325. {
  4326. dst0[0] = src0[0];
  4327. dst0[1] = src0[1];
  4328. src0 += 2;
  4329. dst0 -= stride;
  4330. }
  4331. src0 += srcwgap;
  4332. }
  4333. }
  4334. static void kanna_rotate_8_c3(const unsigned char* src, int srcw, int srch, int srcstride, unsigned char* dst, int /*w*/, int h, int stride)
  4335. {
  4336. const int srcwgap = srcstride - srcw * 3;
  4337. // point to the last dst pixel row
  4338. unsigned char* dstend = dst + stride * (h - 1);
  4339. const unsigned char* src0 = src;
  4340. int y = 0;
  4341. #if __ARM_NEON
  4342. for (; y + 7 < srch; y += 8)
  4343. {
  4344. const unsigned char* src1 = src0 + srcstride;
  4345. unsigned char* dst7 = dstend + y * 3;
  4346. unsigned char* dst6 = dstend + y * 3 - stride;
  4347. int src_step = 2 * srcstride;
  4348. int dst_step = -2 * stride;
  4349. int nn = srcw >> 3;
  4350. int remain = srcw - (nn << 3);
  4351. #if !NCNN_GNU_INLINE_ASM || __aarch64__
  4352. for (; nn > 0; nn--)
  4353. {
  4354. uint8x8x3_t _src0 = vld3_u8(src0);
  4355. uint8x8x3_t _src1 = vld3_u8(src1);
  4356. uint8x8x3_t _src2 = vld3_u8(src0 + src_step);
  4357. uint8x8x3_t _src3 = vld3_u8(src1 + src_step);
  4358. uint8x8x3_t _src4 = vld3_u8(src0 + 2 * src_step);
  4359. uint8x8x3_t _src5 = vld3_u8(src1 + 2 * src_step);
  4360. uint8x8x3_t _src6 = vld3_u8(src0 + 3 * src_step);
  4361. uint8x8x3_t _src7 = vld3_u8(src1 + 3 * src_step);
  4362. uint8x8x2_t _src01t_r = vtrn_u8(_src0.val[0], _src1.val[0]);
  4363. uint8x8x2_t _src23t_r = vtrn_u8(_src2.val[0], _src3.val[0]);
  4364. uint8x8x2_t _src45t_r = vtrn_u8(_src4.val[0], _src5.val[0]);
  4365. uint8x8x2_t _src67t_r = vtrn_u8(_src6.val[0], _src7.val[0]);
  4366. uint8x8x2_t _src01t_g = vtrn_u8(_src0.val[1], _src1.val[1]);
  4367. uint8x8x2_t _src23t_g = vtrn_u8(_src2.val[1], _src3.val[1]);
  4368. uint8x8x2_t _src45t_g = vtrn_u8(_src4.val[1], _src5.val[1]);
  4369. uint8x8x2_t _src67t_g = vtrn_u8(_src6.val[1], _src7.val[1]);
  4370. uint8x8x2_t _src01t_b = vtrn_u8(_src0.val[2], _src1.val[2]);
  4371. uint8x8x2_t _src23t_b = vtrn_u8(_src2.val[2], _src3.val[2]);
  4372. uint8x8x2_t _src45t_b = vtrn_u8(_src4.val[2], _src5.val[2]);
  4373. uint8x8x2_t _src67t_b = vtrn_u8(_src6.val[2], _src7.val[2]);
  4374. uint16x4x2_t _src02tt_r = vtrn_u16(vreinterpret_u16_u8(_src01t_r.val[0]), vreinterpret_u16_u8(_src23t_r.val[0]));
  4375. uint16x4x2_t _src13tt_r = vtrn_u16(vreinterpret_u16_u8(_src01t_r.val[1]), vreinterpret_u16_u8(_src23t_r.val[1]));
  4376. uint16x4x2_t _src46tt_r = vtrn_u16(vreinterpret_u16_u8(_src45t_r.val[0]), vreinterpret_u16_u8(_src67t_r.val[0]));
  4377. uint16x4x2_t _src57tt_r = vtrn_u16(vreinterpret_u16_u8(_src45t_r.val[1]), vreinterpret_u16_u8(_src67t_r.val[1]));
  4378. uint16x4x2_t _src02tt_g = vtrn_u16(vreinterpret_u16_u8(_src01t_g.val[0]), vreinterpret_u16_u8(_src23t_g.val[0]));
  4379. uint16x4x2_t _src13tt_g = vtrn_u16(vreinterpret_u16_u8(_src01t_g.val[1]), vreinterpret_u16_u8(_src23t_g.val[1]));
  4380. uint16x4x2_t _src46tt_g = vtrn_u16(vreinterpret_u16_u8(_src45t_g.val[0]), vreinterpret_u16_u8(_src67t_g.val[0]));
  4381. uint16x4x2_t _src57tt_g = vtrn_u16(vreinterpret_u16_u8(_src45t_g.val[1]), vreinterpret_u16_u8(_src67t_g.val[1]));
  4382. uint16x4x2_t _src02tt_b = vtrn_u16(vreinterpret_u16_u8(_src01t_b.val[0]), vreinterpret_u16_u8(_src23t_b.val[0]));
  4383. uint16x4x2_t _src13tt_b = vtrn_u16(vreinterpret_u16_u8(_src01t_b.val[1]), vreinterpret_u16_u8(_src23t_b.val[1]));
  4384. uint16x4x2_t _src46tt_b = vtrn_u16(vreinterpret_u16_u8(_src45t_b.val[0]), vreinterpret_u16_u8(_src67t_b.val[0]));
  4385. uint16x4x2_t _src57tt_b = vtrn_u16(vreinterpret_u16_u8(_src45t_b.val[1]), vreinterpret_u16_u8(_src67t_b.val[1]));
  4386. uint32x2x2_t _src04ttt_r = vtrn_u32(vreinterpret_u32_u16(_src02tt_r.val[0]), vreinterpret_u32_u16(_src46tt_r.val[0]));
  4387. uint32x2x2_t _src15ttt_r = vtrn_u32(vreinterpret_u32_u16(_src13tt_r.val[0]), vreinterpret_u32_u16(_src57tt_r.val[0]));
  4388. uint32x2x2_t _src26ttt_r = vtrn_u32(vreinterpret_u32_u16(_src02tt_r.val[1]), vreinterpret_u32_u16(_src46tt_r.val[1]));
  4389. uint32x2x2_t _src37ttt_r = vtrn_u32(vreinterpret_u32_u16(_src13tt_r.val[1]), vreinterpret_u32_u16(_src57tt_r.val[1]));
  4390. uint32x2x2_t _src04ttt_g = vtrn_u32(vreinterpret_u32_u16(_src02tt_g.val[0]), vreinterpret_u32_u16(_src46tt_g.val[0]));
  4391. uint32x2x2_t _src15ttt_g = vtrn_u32(vreinterpret_u32_u16(_src13tt_g.val[0]), vreinterpret_u32_u16(_src57tt_g.val[0]));
  4392. uint32x2x2_t _src26ttt_g = vtrn_u32(vreinterpret_u32_u16(_src02tt_g.val[1]), vreinterpret_u32_u16(_src46tt_g.val[1]));
  4393. uint32x2x2_t _src37ttt_g = vtrn_u32(vreinterpret_u32_u16(_src13tt_g.val[1]), vreinterpret_u32_u16(_src57tt_g.val[1]));
  4394. uint32x2x2_t _src04ttt_b = vtrn_u32(vreinterpret_u32_u16(_src02tt_b.val[0]), vreinterpret_u32_u16(_src46tt_b.val[0]));
  4395. uint32x2x2_t _src15ttt_b = vtrn_u32(vreinterpret_u32_u16(_src13tt_b.val[0]), vreinterpret_u32_u16(_src57tt_b.val[0]));
  4396. uint32x2x2_t _src26ttt_b = vtrn_u32(vreinterpret_u32_u16(_src02tt_b.val[1]), vreinterpret_u32_u16(_src46tt_b.val[1]));
  4397. uint32x2x2_t _src37ttt_b = vtrn_u32(vreinterpret_u32_u16(_src13tt_b.val[1]), vreinterpret_u32_u16(_src57tt_b.val[1]));
  4398. uint8x8x3_t _dst0;
  4399. uint8x8x3_t _dst1;
  4400. uint8x8x3_t _dst2;
  4401. uint8x8x3_t _dst3;
  4402. uint8x8x3_t _dst4;
  4403. uint8x8x3_t _dst5;
  4404. uint8x8x3_t _dst6;
  4405. uint8x8x3_t _dst7;
  4406. _dst0.val[0] = vreinterpret_u8_u32(_src04ttt_r.val[0]);
  4407. _dst1.val[0] = vreinterpret_u8_u32(_src15ttt_r.val[0]);
  4408. _dst2.val[0] = vreinterpret_u8_u32(_src26ttt_r.val[0]);
  4409. _dst3.val[0] = vreinterpret_u8_u32(_src37ttt_r.val[0]);
  4410. _dst4.val[0] = vreinterpret_u8_u32(_src04ttt_r.val[1]);
  4411. _dst5.val[0] = vreinterpret_u8_u32(_src15ttt_r.val[1]);
  4412. _dst6.val[0] = vreinterpret_u8_u32(_src26ttt_r.val[1]);
  4413. _dst7.val[0] = vreinterpret_u8_u32(_src37ttt_r.val[1]);
  4414. _dst0.val[1] = vreinterpret_u8_u32(_src04ttt_g.val[0]);
  4415. _dst1.val[1] = vreinterpret_u8_u32(_src15ttt_g.val[0]);
  4416. _dst2.val[1] = vreinterpret_u8_u32(_src26ttt_g.val[0]);
  4417. _dst3.val[1] = vreinterpret_u8_u32(_src37ttt_g.val[0]);
  4418. _dst4.val[1] = vreinterpret_u8_u32(_src04ttt_g.val[1]);
  4419. _dst5.val[1] = vreinterpret_u8_u32(_src15ttt_g.val[1]);
  4420. _dst6.val[1] = vreinterpret_u8_u32(_src26ttt_g.val[1]);
  4421. _dst7.val[1] = vreinterpret_u8_u32(_src37ttt_g.val[1]);
  4422. _dst0.val[2] = vreinterpret_u8_u32(_src04ttt_b.val[0]);
  4423. _dst1.val[2] = vreinterpret_u8_u32(_src15ttt_b.val[0]);
  4424. _dst2.val[2] = vreinterpret_u8_u32(_src26ttt_b.val[0]);
  4425. _dst3.val[2] = vreinterpret_u8_u32(_src37ttt_b.val[0]);
  4426. _dst4.val[2] = vreinterpret_u8_u32(_src04ttt_b.val[1]);
  4427. _dst5.val[2] = vreinterpret_u8_u32(_src15ttt_b.val[1]);
  4428. _dst6.val[2] = vreinterpret_u8_u32(_src26ttt_b.val[1]);
  4429. _dst7.val[2] = vreinterpret_u8_u32(_src37ttt_b.val[1]);
  4430. vst3_u8(dst7, _dst0);
  4431. vst3_u8(dst6, _dst1);
  4432. vst3_u8(dst7 + dst_step, _dst2);
  4433. vst3_u8(dst6 + dst_step, _dst3);
  4434. vst3_u8(dst7 + 2 * dst_step, _dst4);
  4435. vst3_u8(dst6 + 2 * dst_step, _dst5);
  4436. vst3_u8(dst7 + 3 * dst_step, _dst6);
  4437. vst3_u8(dst6 + 3 * dst_step, _dst7);
  4438. src0 += 3 * 8;
  4439. src1 += 3 * 8;
  4440. dst7 += 4 * dst_step;
  4441. dst6 += 4 * dst_step;
  4442. }
  4443. #else
  4444. if (nn > 0)
  4445. {
  4446. asm volatile(
  4447. "0: \n"
  4448. "pld [%1, #192] \n"
  4449. "vld3.u8 {d0-d2}, [%1], %10 \n"
  4450. "pld [%2, #192] \n"
  4451. "vld3.u8 {d4-d6}, [%2], %10 \n"
  4452. "pld [%1, #192] \n"
  4453. "vld3.u8 {d8-d10}, [%1], %10 \n"
  4454. "vtrn.u8 q0, q2 \n" // _src01t_r
  4455. "vtrn.u8 d2, d6 \n"
  4456. "pld [%2, #192] \n"
  4457. "vld3.u8 {d12-d14}, [%2], %10\n"
  4458. "pld [%1, #192] \n"
  4459. "vld3.u8 {d16-d18}, [%1], %10\n"
  4460. "vtrn.u8 q4, q6 \n" // _src23t_r
  4461. "vtrn.u8 d10, d14 \n"
  4462. "pld [%2, #192] \n"
  4463. "vld3.u8 {d20-d22}, [%2], %10\n"
  4464. "pld [%1, #192] \n"
  4465. "vld3.u8 {d24-d26}, [%1], %10\n"
  4466. "vtrn.u8 q8, q10 \n" // _src45t_r
  4467. "vtrn.u8 d18, d22 \n"
  4468. "pld [%2, #192] \n"
  4469. "vld3.u8 {d28-d30}, [%2], %10\n"
  4470. "vtrn.u8 q12, q14 \n" // _src67t_r
  4471. "vtrn.u8 d26, d30 \n"
  4472. "sub %1, %1, %10, lsl #2 \n" // restore src0
  4473. "vtrn.u16 q0, q4 \n" // _src02tt_r
  4474. "vtrn.u16 d2, d10 \n"
  4475. "sub %2, %2, %10, lsl #2 \n" // restore src1
  4476. "vtrn.u16 q2, q6 \n" // _src13tt_r
  4477. "vtrn.u16 d6, d14 \n"
  4478. "add %1, #24 \n" // src0 += 24
  4479. "vtrn.u16 q8, q12 \n" // _src46tt_r
  4480. "vtrn.u16 d18, d26 \n"
  4481. "add %2, #24 \n" // src1 += 24
  4482. "vtrn.u16 q10, q14 \n" // _src57tt_r
  4483. "vtrn.u16 d22, d30 \n"
  4484. "vtrn.u32 q0, q8 \n" // _src04ttt_r
  4485. "vtrn.u32 d2, d18 \n"
  4486. "vtrn.u32 q2, q10 \n" // _src15ttt_r
  4487. "vst3.u8 {d0-d2}, [%3], %11 \n"
  4488. "vtrn.u32 d6, d22 \n"
  4489. "vtrn.u32 q4, q12 \n" // _src26ttt_r
  4490. "vst3.u8 {d4-d6}, [%4], %11 \n"
  4491. "vtrn.u32 d10, d26 \n"
  4492. "vtrn.u32 q6, q14 \n" // _src37ttt_r
  4493. "vst3.u8 {d8-d10}, [%3], %11 \n"
  4494. "vtrn.u32 d14, d30 \n"
  4495. "subs %0, #1 \n"
  4496. "vst3.u8 {d16-d18}, [%3], %11\n"
  4497. "vst3.u8 {d12-d14}, [%4], %11\n"
  4498. "vst3.u8 {d20-d22}, [%4], %11\n"
  4499. "vst3.u8 {d24-d26}, [%3], %11\n"
  4500. "vst3.u8 {d28-d30}, [%4], %11\n"
  4501. "bne 0b \n"
  4502. : "=r"(nn), // %0
  4503. "=r"(src0), // %1
  4504. "=r"(src1), // %2
  4505. "=r"(dst7), // %3
  4506. "=r"(dst6) // %4
  4507. : "0"(nn),
  4508. "1"(src0),
  4509. "2"(src1),
  4510. "3"(dst7),
  4511. "4"(dst6),
  4512. "r"(src_step), // %10
  4513. "r"(dst_step) // %11
  4514. : "cc", "memory", "q0", "q1", "q2", "q3", "q4", "q5", "q6", "q7", "q8", "q9", "q10", "q11", "q12", "q13", "q14", "q15");
  4515. }
  4516. #endif // __aarch64__
  4517. for (; remain > 0; remain--)
  4518. {
  4519. dst7[0] = src0[0];
  4520. dst7[1] = src0[1];
  4521. dst7[2] = src0[2];
  4522. dst7[3] = src1[0];
  4523. dst7[4] = src1[1];
  4524. dst7[5] = src1[2];
  4525. dst7[6] = src0[0 + src_step];
  4526. dst7[7] = src0[1 + src_step];
  4527. dst7[8] = src0[2 + src_step];
  4528. dst7[9] = src1[0 + src_step];
  4529. dst7[10] = src1[1 + src_step];
  4530. dst7[11] = src1[2 + src_step];
  4531. dst7[12] = src0[0 + 2 * src_step];
  4532. dst7[13] = src0[1 + 2 * src_step];
  4533. dst7[14] = src0[2 + 2 * src_step];
  4534. dst7[15] = src1[0 + 2 * src_step];
  4535. dst7[16] = src1[1 + 2 * src_step];
  4536. dst7[17] = src1[2 + 2 * src_step];
  4537. dst7[18] = src0[0 + 3 * src_step];
  4538. dst7[19] = src0[1 + 3 * src_step];
  4539. dst7[20] = src0[2 + 3 * src_step];
  4540. dst7[21] = src1[0 + 3 * src_step];
  4541. dst7[22] = src1[1 + 3 * src_step];
  4542. dst7[23] = src1[2 + 3 * src_step];
  4543. src0 += 3;
  4544. src1 += 3;
  4545. dst7 -= stride;
  4546. }
  4547. src0 += srcwgap + 7 * srcstride;
  4548. }
  4549. #endif // __ARM_NEON
  4550. for (; y < srch; y++)
  4551. {
  4552. unsigned char* dst0 = dstend + y * 3;
  4553. int x = 0;
  4554. for (; x < srcw; x++)
  4555. {
  4556. dst0[0] = src0[0];
  4557. dst0[1] = src0[1];
  4558. dst0[2] = src0[2];
  4559. src0 += 3;
  4560. dst0 -= stride;
  4561. }
  4562. src0 += srcwgap;
  4563. }
  4564. }
  4565. static void kanna_rotate_8_c4(const unsigned char* src, int srcw, int srch, int srcstride, unsigned char* dst, int /*w*/, int h, int stride)
  4566. {
  4567. const int srcwgap = srcstride - srcw * 4;
  4568. // point to the last dst pixel row
  4569. unsigned char* dstend = dst + stride * (h - 1);
  4570. const unsigned char* src0 = src;
  4571. int y = 0;
  4572. #if __ARM_NEON
  4573. for (; y + 7 < srch; y += 8)
  4574. {
  4575. const unsigned char* src1 = src0 + srcstride;
  4576. unsigned char* dst7 = dstend + y * 4;
  4577. unsigned char* dst6 = dstend + y * 4 - stride;
  4578. int src_step = 2 * srcstride;
  4579. int dst_step = -2 * stride;
  4580. int nn = srcw >> 3;
  4581. int remain = srcw - (nn << 3);
  4582. #if !NCNN_GNU_INLINE_ASM || __aarch64__
  4583. for (; nn > 0; nn--)
  4584. {
  4585. uint8x8x4_t _src0 = vld4_u8(src0);
  4586. uint8x8x4_t _src1 = vld4_u8(src1);
  4587. uint8x8x4_t _src2 = vld4_u8(src0 + src_step);
  4588. uint8x8x4_t _src3 = vld4_u8(src1 + src_step);
  4589. uint8x8x4_t _src4 = vld4_u8(src0 + 2 * src_step);
  4590. uint8x8x4_t _src5 = vld4_u8(src1 + 2 * src_step);
  4591. uint8x8x4_t _src6 = vld4_u8(src0 + 3 * src_step);
  4592. uint8x8x4_t _src7 = vld4_u8(src1 + 3 * src_step);
  4593. uint8x8x2_t _src01t_r = vtrn_u8(_src0.val[0], _src1.val[0]);
  4594. uint8x8x2_t _src23t_r = vtrn_u8(_src2.val[0], _src3.val[0]);
  4595. uint8x8x2_t _src45t_r = vtrn_u8(_src4.val[0], _src5.val[0]);
  4596. uint8x8x2_t _src67t_r = vtrn_u8(_src6.val[0], _src7.val[0]);
  4597. uint8x8x2_t _src01t_g = vtrn_u8(_src0.val[1], _src1.val[1]);
  4598. uint8x8x2_t _src23t_g = vtrn_u8(_src2.val[1], _src3.val[1]);
  4599. uint8x8x2_t _src45t_g = vtrn_u8(_src4.val[1], _src5.val[1]);
  4600. uint8x8x2_t _src67t_g = vtrn_u8(_src6.val[1], _src7.val[1]);
  4601. uint8x8x2_t _src01t_b = vtrn_u8(_src0.val[2], _src1.val[2]);
  4602. uint8x8x2_t _src23t_b = vtrn_u8(_src2.val[2], _src3.val[2]);
  4603. uint8x8x2_t _src45t_b = vtrn_u8(_src4.val[2], _src5.val[2]);
  4604. uint8x8x2_t _src67t_b = vtrn_u8(_src6.val[2], _src7.val[2]);
  4605. uint8x8x2_t _src01t_a = vtrn_u8(_src0.val[3], _src1.val[3]);
  4606. uint8x8x2_t _src23t_a = vtrn_u8(_src2.val[3], _src3.val[3]);
  4607. uint8x8x2_t _src45t_a = vtrn_u8(_src4.val[3], _src5.val[3]);
  4608. uint8x8x2_t _src67t_a = vtrn_u8(_src6.val[3], _src7.val[3]);
  4609. uint16x4x2_t _src02tt_r = vtrn_u16(vreinterpret_u16_u8(_src01t_r.val[0]), vreinterpret_u16_u8(_src23t_r.val[0]));
  4610. uint16x4x2_t _src13tt_r = vtrn_u16(vreinterpret_u16_u8(_src01t_r.val[1]), vreinterpret_u16_u8(_src23t_r.val[1]));
  4611. uint16x4x2_t _src46tt_r = vtrn_u16(vreinterpret_u16_u8(_src45t_r.val[0]), vreinterpret_u16_u8(_src67t_r.val[0]));
  4612. uint16x4x2_t _src57tt_r = vtrn_u16(vreinterpret_u16_u8(_src45t_r.val[1]), vreinterpret_u16_u8(_src67t_r.val[1]));
  4613. uint16x4x2_t _src02tt_g = vtrn_u16(vreinterpret_u16_u8(_src01t_g.val[0]), vreinterpret_u16_u8(_src23t_g.val[0]));
  4614. uint16x4x2_t _src13tt_g = vtrn_u16(vreinterpret_u16_u8(_src01t_g.val[1]), vreinterpret_u16_u8(_src23t_g.val[1]));
  4615. uint16x4x2_t _src46tt_g = vtrn_u16(vreinterpret_u16_u8(_src45t_g.val[0]), vreinterpret_u16_u8(_src67t_g.val[0]));
  4616. uint16x4x2_t _src57tt_g = vtrn_u16(vreinterpret_u16_u8(_src45t_g.val[1]), vreinterpret_u16_u8(_src67t_g.val[1]));
  4617. uint16x4x2_t _src02tt_b = vtrn_u16(vreinterpret_u16_u8(_src01t_b.val[0]), vreinterpret_u16_u8(_src23t_b.val[0]));
  4618. uint16x4x2_t _src13tt_b = vtrn_u16(vreinterpret_u16_u8(_src01t_b.val[1]), vreinterpret_u16_u8(_src23t_b.val[1]));
  4619. uint16x4x2_t _src46tt_b = vtrn_u16(vreinterpret_u16_u8(_src45t_b.val[0]), vreinterpret_u16_u8(_src67t_b.val[0]));
  4620. uint16x4x2_t _src57tt_b = vtrn_u16(vreinterpret_u16_u8(_src45t_b.val[1]), vreinterpret_u16_u8(_src67t_b.val[1]));
  4621. uint16x4x2_t _src02tt_a = vtrn_u16(vreinterpret_u16_u8(_src01t_a.val[0]), vreinterpret_u16_u8(_src23t_a.val[0]));
  4622. uint16x4x2_t _src13tt_a = vtrn_u16(vreinterpret_u16_u8(_src01t_a.val[1]), vreinterpret_u16_u8(_src23t_a.val[1]));
  4623. uint16x4x2_t _src46tt_a = vtrn_u16(vreinterpret_u16_u8(_src45t_a.val[0]), vreinterpret_u16_u8(_src67t_a.val[0]));
  4624. uint16x4x2_t _src57tt_a = vtrn_u16(vreinterpret_u16_u8(_src45t_a.val[1]), vreinterpret_u16_u8(_src67t_a.val[1]));
  4625. uint32x2x2_t _src04ttt_r = vtrn_u32(vreinterpret_u32_u16(_src02tt_r.val[0]), vreinterpret_u32_u16(_src46tt_r.val[0]));
  4626. uint32x2x2_t _src15ttt_r = vtrn_u32(vreinterpret_u32_u16(_src13tt_r.val[0]), vreinterpret_u32_u16(_src57tt_r.val[0]));
  4627. uint32x2x2_t _src26ttt_r = vtrn_u32(vreinterpret_u32_u16(_src02tt_r.val[1]), vreinterpret_u32_u16(_src46tt_r.val[1]));
  4628. uint32x2x2_t _src37ttt_r = vtrn_u32(vreinterpret_u32_u16(_src13tt_r.val[1]), vreinterpret_u32_u16(_src57tt_r.val[1]));
  4629. uint32x2x2_t _src04ttt_g = vtrn_u32(vreinterpret_u32_u16(_src02tt_g.val[0]), vreinterpret_u32_u16(_src46tt_g.val[0]));
  4630. uint32x2x2_t _src15ttt_g = vtrn_u32(vreinterpret_u32_u16(_src13tt_g.val[0]), vreinterpret_u32_u16(_src57tt_g.val[0]));
  4631. uint32x2x2_t _src26ttt_g = vtrn_u32(vreinterpret_u32_u16(_src02tt_g.val[1]), vreinterpret_u32_u16(_src46tt_g.val[1]));
  4632. uint32x2x2_t _src37ttt_g = vtrn_u32(vreinterpret_u32_u16(_src13tt_g.val[1]), vreinterpret_u32_u16(_src57tt_g.val[1]));
  4633. uint32x2x2_t _src04ttt_b = vtrn_u32(vreinterpret_u32_u16(_src02tt_b.val[0]), vreinterpret_u32_u16(_src46tt_b.val[0]));
  4634. uint32x2x2_t _src15ttt_b = vtrn_u32(vreinterpret_u32_u16(_src13tt_b.val[0]), vreinterpret_u32_u16(_src57tt_b.val[0]));
  4635. uint32x2x2_t _src26ttt_b = vtrn_u32(vreinterpret_u32_u16(_src02tt_b.val[1]), vreinterpret_u32_u16(_src46tt_b.val[1]));
  4636. uint32x2x2_t _src37ttt_b = vtrn_u32(vreinterpret_u32_u16(_src13tt_b.val[1]), vreinterpret_u32_u16(_src57tt_b.val[1]));
  4637. uint32x2x2_t _src04ttt_a = vtrn_u32(vreinterpret_u32_u16(_src02tt_a.val[0]), vreinterpret_u32_u16(_src46tt_a.val[0]));
  4638. uint32x2x2_t _src15ttt_a = vtrn_u32(vreinterpret_u32_u16(_src13tt_a.val[0]), vreinterpret_u32_u16(_src57tt_a.val[0]));
  4639. uint32x2x2_t _src26ttt_a = vtrn_u32(vreinterpret_u32_u16(_src02tt_a.val[1]), vreinterpret_u32_u16(_src46tt_a.val[1]));
  4640. uint32x2x2_t _src37ttt_a = vtrn_u32(vreinterpret_u32_u16(_src13tt_a.val[1]), vreinterpret_u32_u16(_src57tt_a.val[1]));
  4641. uint8x8x4_t _dst0;
  4642. uint8x8x4_t _dst1;
  4643. uint8x8x4_t _dst2;
  4644. uint8x8x4_t _dst3;
  4645. uint8x8x4_t _dst4;
  4646. uint8x8x4_t _dst5;
  4647. uint8x8x4_t _dst6;
  4648. uint8x8x4_t _dst7;
  4649. _dst0.val[0] = vreinterpret_u8_u32(_src04ttt_r.val[0]);
  4650. _dst1.val[0] = vreinterpret_u8_u32(_src15ttt_r.val[0]);
  4651. _dst2.val[0] = vreinterpret_u8_u32(_src26ttt_r.val[0]);
  4652. _dst3.val[0] = vreinterpret_u8_u32(_src37ttt_r.val[0]);
  4653. _dst4.val[0] = vreinterpret_u8_u32(_src04ttt_r.val[1]);
  4654. _dst5.val[0] = vreinterpret_u8_u32(_src15ttt_r.val[1]);
  4655. _dst6.val[0] = vreinterpret_u8_u32(_src26ttt_r.val[1]);
  4656. _dst7.val[0] = vreinterpret_u8_u32(_src37ttt_r.val[1]);
  4657. _dst0.val[1] = vreinterpret_u8_u32(_src04ttt_g.val[0]);
  4658. _dst1.val[1] = vreinterpret_u8_u32(_src15ttt_g.val[0]);
  4659. _dst2.val[1] = vreinterpret_u8_u32(_src26ttt_g.val[0]);
  4660. _dst3.val[1] = vreinterpret_u8_u32(_src37ttt_g.val[0]);
  4661. _dst4.val[1] = vreinterpret_u8_u32(_src04ttt_g.val[1]);
  4662. _dst5.val[1] = vreinterpret_u8_u32(_src15ttt_g.val[1]);
  4663. _dst6.val[1] = vreinterpret_u8_u32(_src26ttt_g.val[1]);
  4664. _dst7.val[1] = vreinterpret_u8_u32(_src37ttt_g.val[1]);
  4665. _dst0.val[2] = vreinterpret_u8_u32(_src04ttt_b.val[0]);
  4666. _dst1.val[2] = vreinterpret_u8_u32(_src15ttt_b.val[0]);
  4667. _dst2.val[2] = vreinterpret_u8_u32(_src26ttt_b.val[0]);
  4668. _dst3.val[2] = vreinterpret_u8_u32(_src37ttt_b.val[0]);
  4669. _dst4.val[2] = vreinterpret_u8_u32(_src04ttt_b.val[1]);
  4670. _dst5.val[2] = vreinterpret_u8_u32(_src15ttt_b.val[1]);
  4671. _dst6.val[2] = vreinterpret_u8_u32(_src26ttt_b.val[1]);
  4672. _dst7.val[2] = vreinterpret_u8_u32(_src37ttt_b.val[1]);
  4673. _dst0.val[3] = vreinterpret_u8_u32(_src04ttt_a.val[0]);
  4674. _dst1.val[3] = vreinterpret_u8_u32(_src15ttt_a.val[0]);
  4675. _dst2.val[3] = vreinterpret_u8_u32(_src26ttt_a.val[0]);
  4676. _dst3.val[3] = vreinterpret_u8_u32(_src37ttt_a.val[0]);
  4677. _dst4.val[3] = vreinterpret_u8_u32(_src04ttt_a.val[1]);
  4678. _dst5.val[3] = vreinterpret_u8_u32(_src15ttt_a.val[1]);
  4679. _dst6.val[3] = vreinterpret_u8_u32(_src26ttt_a.val[1]);
  4680. _dst7.val[3] = vreinterpret_u8_u32(_src37ttt_a.val[1]);
  4681. vst4_u8(dst7, _dst0);
  4682. vst4_u8(dst6, _dst1);
  4683. vst4_u8(dst7 + dst_step, _dst2);
  4684. vst4_u8(dst6 + dst_step, _dst3);
  4685. vst4_u8(dst7 + 2 * dst_step, _dst4);
  4686. vst4_u8(dst6 + 2 * dst_step, _dst5);
  4687. vst4_u8(dst7 + 3 * dst_step, _dst6);
  4688. vst4_u8(dst6 + 3 * dst_step, _dst7);
  4689. src0 += 4 * 8;
  4690. src1 += 4 * 8;
  4691. dst7 += 4 * dst_step;
  4692. dst6 += 4 * dst_step;
  4693. }
  4694. #else
  4695. if (nn > 0)
  4696. {
  4697. asm volatile(
  4698. "0: \n"
  4699. "pld [%1, #256] \n"
  4700. "vld4.u8 {d0-d3}, [%1], %10 \n"
  4701. "pld [%2, #256] \n"
  4702. "vld4.u8 {d4-d7}, [%2], %10 \n"
  4703. "pld [%1, #256] \n"
  4704. "vld4.u8 {d8-d11}, [%1], %10 \n"
  4705. "vtrn.u8 q0, q2 \n" // _src01t_r
  4706. "vtrn.u8 q1, q3 \n"
  4707. "pld [%2, #256] \n"
  4708. "vld4.u8 {d12-d15}, [%2], %10\n"
  4709. "pld [%1, #256] \n"
  4710. "vld4.u8 {d16-d19}, [%1], %10\n"
  4711. "vtrn.u8 q4, q6 \n" // _src23t_r
  4712. "vtrn.u8 q5, q7 \n"
  4713. "pld [%2, #256] \n"
  4714. "vld4.u8 {d20-d23}, [%2], %10\n"
  4715. "pld [%1, #256] \n"
  4716. "vld4.u8 {d24-d27}, [%1], %10\n"
  4717. "vtrn.u8 q8, q10 \n" // _src45t_r
  4718. "vtrn.u8 q9, q11 \n"
  4719. "pld [%2, #256] \n"
  4720. "vld4.u8 {d28-d31}, [%2], %10\n"
  4721. "vtrn.u8 q12, q14 \n" // _src67t_r
  4722. "vtrn.u8 q13, q15 \n"
  4723. "sub %1, %1, %10, lsl #2 \n" // restore src0
  4724. "vtrn.u16 q0, q4 \n" // _src02tt_r
  4725. "vtrn.u16 q1, q5 \n"
  4726. "sub %2, %2, %10, lsl #2 \n" // restore src1
  4727. "vtrn.u16 q2, q6 \n" // _src13tt_r
  4728. "vtrn.u16 q3, q7 \n"
  4729. "add %1, #32 \n" // src0 += 32
  4730. "vtrn.u16 q8, q12 \n" // _src46tt_r
  4731. "vtrn.u16 q9, q13 \n"
  4732. "add %2, #32 \n" // src1 += 32
  4733. "vtrn.u16 q10, q14 \n" // _src57tt_r
  4734. "vtrn.u16 q11, q15 \n"
  4735. "vtrn.u32 q0, q8 \n" // _src04ttt_r
  4736. "vtrn.u32 q1, q9 \n"
  4737. "vtrn.u32 q2, q10 \n" // _src15ttt_r
  4738. "vst4.u8 {d0-d3}, [%3], %11 \n"
  4739. "vtrn.u32 q3, q11 \n"
  4740. "vtrn.u32 q4, q12 \n" // _src26ttt_r
  4741. "vst4.u8 {d4-d7}, [%4], %11 \n"
  4742. "vtrn.u32 q5, q13 \n"
  4743. "vtrn.u32 q6, q14 \n" // _src37ttt_r
  4744. "vst4.u8 {d8-d11}, [%3], %11 \n"
  4745. "vtrn.u32 q7, q15 \n"
  4746. "subs %0, #1 \n"
  4747. "vst4.u8 {d16-d19}, [%3], %11\n"
  4748. "vst4.u8 {d12-d15}, [%4], %11\n"
  4749. "vst4.u8 {d20-d23}, [%4], %11\n"
  4750. "vst4.u8 {d24-d27}, [%3], %11\n"
  4751. "vst4.u8 {d28-d31}, [%4], %11\n"
  4752. "bne 0b \n"
  4753. : "=r"(nn), // %0
  4754. "=r"(src0), // %1
  4755. "=r"(src1), // %2
  4756. "=r"(dst7), // %3
  4757. "=r"(dst6) // %4
  4758. : "0"(nn),
  4759. "1"(src0),
  4760. "2"(src1),
  4761. "3"(dst7),
  4762. "4"(dst6),
  4763. "r"(src_step), // %10
  4764. "r"(dst_step) // %11
  4765. : "cc", "memory", "q0", "q1", "q2", "q3", "q4", "q5", "q6", "q7", "q8", "q9", "q10", "q11", "q12", "q13", "q14", "q15");
  4766. }
  4767. #endif // __aarch64__
  4768. for (; remain > 0; remain--)
  4769. {
  4770. dst7[0] = src0[0];
  4771. dst7[1] = src0[1];
  4772. dst7[2] = src0[2];
  4773. dst7[3] = src0[3];
  4774. dst7[4] = src1[0];
  4775. dst7[5] = src1[1];
  4776. dst7[6] = src1[2];
  4777. dst7[7] = src1[3];
  4778. dst7[8] = src0[0 + src_step];
  4779. dst7[9] = src0[1 + src_step];
  4780. dst7[10] = src0[2 + src_step];
  4781. dst7[11] = src0[3 + src_step];
  4782. dst7[12] = src1[0 + src_step];
  4783. dst7[13] = src1[1 + src_step];
  4784. dst7[14] = src1[2 + src_step];
  4785. dst7[15] = src1[3 + src_step];
  4786. dst7[16] = src0[0 + 2 * src_step];
  4787. dst7[17] = src0[1 + 2 * src_step];
  4788. dst7[18] = src0[2 + 2 * src_step];
  4789. dst7[19] = src0[3 + 2 * src_step];
  4790. dst7[20] = src1[0 + 2 * src_step];
  4791. dst7[21] = src1[1 + 2 * src_step];
  4792. dst7[22] = src1[2 + 2 * src_step];
  4793. dst7[23] = src1[3 + 2 * src_step];
  4794. dst7[24] = src0[0 + 3 * src_step];
  4795. dst7[25] = src0[1 + 3 * src_step];
  4796. dst7[26] = src0[2 + 3 * src_step];
  4797. dst7[27] = src0[3 + 3 * src_step];
  4798. dst7[28] = src1[0 + 3 * src_step];
  4799. dst7[29] = src1[1 + 3 * src_step];
  4800. dst7[30] = src1[2 + 3 * src_step];
  4801. dst7[31] = src1[3 + 3 * src_step];
  4802. src0 += 4;
  4803. src1 += 4;
  4804. dst7 -= stride;
  4805. }
  4806. src0 += srcwgap + 7 * srcstride;
  4807. }
  4808. #endif // __ARM_NEON
  4809. for (; y < srch; y++)
  4810. {
  4811. unsigned char* dst0 = dstend + y * 4;
  4812. int x = 0;
  4813. for (; x < srcw; x++)
  4814. {
  4815. dst0[0] = src0[0];
  4816. dst0[1] = src0[1];
  4817. dst0[2] = src0[2];
  4818. dst0[3] = src0[3];
  4819. src0 += 4;
  4820. dst0 -= stride;
  4821. }
  4822. src0 += srcwgap;
  4823. }
  4824. }
  4825. void kanna_rotate_c1(const unsigned char* src, int srcw, int srch, unsigned char* dst, int w, int h, int type)
  4826. {
  4827. return kanna_rotate_c1(src, srcw, srch, srcw, dst, w, h, w, type);
  4828. }
  4829. void kanna_rotate_c2(const unsigned char* src, int srcw, int srch, unsigned char* dst, int w, int h, int type)
  4830. {
  4831. return kanna_rotate_c2(src, srcw, srch, srcw * 2, dst, w, h, w * 2, type);
  4832. }
  4833. void kanna_rotate_c3(const unsigned char* src, int srcw, int srch, unsigned char* dst, int w, int h, int type)
  4834. {
  4835. return kanna_rotate_c3(src, srcw, srch, srcw * 3, dst, w, h, w * 3, type);
  4836. }
  4837. void kanna_rotate_c4(const unsigned char* src, int srcw, int srch, unsigned char* dst, int w, int h, int type)
  4838. {
  4839. return kanna_rotate_c4(src, srcw, srch, srcw * 4, dst, w, h, w * 4, type);
  4840. }
  4841. void kanna_rotate_c1(const unsigned char* src, int srcw, int srch, int srcstride, unsigned char* dst, int w, int h, int stride, int type)
  4842. {
  4843. // assert srcw == w && srch == h for type 1234
  4844. // assert srcw == h && srch == w for type 5678
  4845. switch (type)
  4846. {
  4847. case 1:
  4848. kanna_rotate_1_c1(src, srcw, srch, srcstride, dst, w, h, stride);
  4849. break;
  4850. case 2:
  4851. kanna_rotate_2_c1(src, srcw, srch, srcstride, dst, w, h, stride);
  4852. break;
  4853. case 3:
  4854. kanna_rotate_3_c1(src, srcw, srch, srcstride, dst, w, h, stride);
  4855. break;
  4856. case 4:
  4857. kanna_rotate_4_c1(src, srcw, srch, srcstride, dst, w, h, stride);
  4858. break;
  4859. case 5:
  4860. kanna_rotate_5_c1(src, srcw, srch, srcstride, dst, w, h, stride);
  4861. break;
  4862. case 6:
  4863. kanna_rotate_6_c1(src, srcw, srch, srcstride, dst, w, h, stride);
  4864. break;
  4865. case 7:
  4866. kanna_rotate_7_c1(src, srcw, srch, srcstride, dst, w, h, stride);
  4867. break;
  4868. case 8:
  4869. kanna_rotate_8_c1(src, srcw, srch, srcstride, dst, w, h, stride);
  4870. break;
  4871. default:
  4872. // unsupported rotate type
  4873. break;
  4874. }
  4875. }
  4876. void kanna_rotate_c2(const unsigned char* src, int srcw, int srch, int srcstride, unsigned char* dst, int w, int h, int stride, int type)
  4877. {
  4878. // assert srcw == w && srch == h for type 1234
  4879. // assert srcw == h && srch == w for type 5678
  4880. switch (type)
  4881. {
  4882. case 1:
  4883. kanna_rotate_1_c2(src, srcw, srch, srcstride, dst, w, h, stride);
  4884. break;
  4885. case 2:
  4886. kanna_rotate_2_c2(src, srcw, srch, srcstride, dst, w, h, stride);
  4887. break;
  4888. case 3:
  4889. kanna_rotate_3_c2(src, srcw, srch, srcstride, dst, w, h, stride);
  4890. break;
  4891. case 4:
  4892. kanna_rotate_4_c2(src, srcw, srch, srcstride, dst, w, h, stride);
  4893. break;
  4894. case 5:
  4895. kanna_rotate_5_c2(src, srcw, srch, srcstride, dst, w, h, stride);
  4896. break;
  4897. case 6:
  4898. kanna_rotate_6_c2(src, srcw, srch, srcstride, dst, w, h, stride);
  4899. break;
  4900. case 7:
  4901. kanna_rotate_7_c2(src, srcw, srch, srcstride, dst, w, h, stride);
  4902. break;
  4903. case 8:
  4904. kanna_rotate_8_c2(src, srcw, srch, srcstride, dst, w, h, stride);
  4905. break;
  4906. default:
  4907. // unsupported rotate type
  4908. break;
  4909. }
  4910. }
  4911. void kanna_rotate_c3(const unsigned char* src, int srcw, int srch, int srcstride, unsigned char* dst, int w, int h, int stride, int type)
  4912. {
  4913. // assert srcw == w && srch == h for type 1234
  4914. // assert srcw == h && srch == w for type 5678
  4915. switch (type)
  4916. {
  4917. case 1:
  4918. kanna_rotate_1_c3(src, srcw, srch, srcstride, dst, w, h, stride);
  4919. break;
  4920. case 2:
  4921. kanna_rotate_2_c3(src, srcw, srch, srcstride, dst, w, h, stride);
  4922. break;
  4923. case 3:
  4924. kanna_rotate_3_c3(src, srcw, srch, srcstride, dst, w, h, stride);
  4925. break;
  4926. case 4:
  4927. kanna_rotate_4_c3(src, srcw, srch, srcstride, dst, w, h, stride);
  4928. break;
  4929. case 5:
  4930. kanna_rotate_5_c3(src, srcw, srch, srcstride, dst, w, h, stride);
  4931. break;
  4932. case 6:
  4933. kanna_rotate_6_c3(src, srcw, srch, srcstride, dst, w, h, stride);
  4934. break;
  4935. case 7:
  4936. kanna_rotate_7_c3(src, srcw, srch, srcstride, dst, w, h, stride);
  4937. break;
  4938. case 8:
  4939. kanna_rotate_8_c3(src, srcw, srch, srcstride, dst, w, h, stride);
  4940. break;
  4941. default:
  4942. // unsupported rotate type
  4943. break;
  4944. }
  4945. }
  4946. void kanna_rotate_c4(const unsigned char* src, int srcw, int srch, int srcstride, unsigned char* dst, int w, int h, int stride, int type)
  4947. {
  4948. // assert srcw == w && srch == h for type 1234
  4949. // assert srcw == h && srch == w for type 5678
  4950. switch (type)
  4951. {
  4952. case 1:
  4953. kanna_rotate_1_c4(src, srcw, srch, srcstride, dst, w, h, stride);
  4954. break;
  4955. case 2:
  4956. kanna_rotate_2_c4(src, srcw, srch, srcstride, dst, w, h, stride);
  4957. break;
  4958. case 3:
  4959. kanna_rotate_3_c4(src, srcw, srch, srcstride, dst, w, h, stride);
  4960. break;
  4961. case 4:
  4962. kanna_rotate_4_c4(src, srcw, srch, srcstride, dst, w, h, stride);
  4963. break;
  4964. case 5:
  4965. kanna_rotate_5_c4(src, srcw, srch, srcstride, dst, w, h, stride);
  4966. break;
  4967. case 6:
  4968. kanna_rotate_6_c4(src, srcw, srch, srcstride, dst, w, h, stride);
  4969. break;
  4970. case 7:
  4971. kanna_rotate_7_c4(src, srcw, srch, srcstride, dst, w, h, stride);
  4972. break;
  4973. case 8:
  4974. kanna_rotate_8_c4(src, srcw, srch, srcstride, dst, w, h, stride);
  4975. break;
  4976. default:
  4977. // unsupported rotate type
  4978. break;
  4979. }
  4980. }
  4981. void kanna_rotate_yuv420sp(const unsigned char* src, int srcw, int srch, unsigned char* dst, int w, int h, int type)
  4982. {
  4983. // assert srcw % 2 == 0
  4984. // assert srch % 2 == 0
  4985. // assert w % 2 == 0
  4986. // assert h % 2 == 0
  4987. const unsigned char* srcY = src;
  4988. unsigned char* dstY = dst;
  4989. kanna_rotate_c1(srcY, srcw, srch, dstY, w, h, type);
  4990. const unsigned char* srcUV = src + srcw * srch;
  4991. unsigned char* dstUV = dst + w * h;
  4992. kanna_rotate_c2(srcUV, srcw / 2, srch / 2, dstUV, w / 2, h / 2, type);
  4993. }
  4994. #endif // NCNN_PIXEL_ROTATE
  4995. } // namespace ncnn