diff --git a/ansible/group_vars/all/vault b/ansible/group_vars/all/vault index d9ff980..7d47605 100644 --- a/ansible/group_vars/all/vault +++ b/ansible/group_vars/all/vault @@ -1,418 +1,424 @@ $ANSIBLE_VAULT;1.1;AES256 -39633233363564393465633065323730343130653362373436356334323930393036656638303033 -6333636536643835303936633039633138336434663565320a643961663732373630333238633466 -62633236666331326165653637353038313163643162323763313635306230333338613164316337 -6133396638336465350a306163643138653639646133346561323333313332336639393764343364 -30666535323264363661653236333839643631376337303563383739656238343732396364366437 -33333437643930656564376236333736336465356134653239393036313830333462363633653366 -62623863613261323135633262633066383363313731646361326232366661356261316633323164 -34653661383366643531353939303333363832323731623830393262373538613631613631616463 -66396163613965343833356432643534393539643961373263323533316664386665383330326665 -35393537363734333961393931386161386234323264316466303361663139323431303034393035 -33353432363162353261646231313136643533363563343537666565306638353564646235623835 -33653663393838303364363762613663646139326135313438663838323064326265303733346164 -62386130613134653739353930623330663135633632346531306535643332303939356263636232 -34313939333263313133373635396138643838396235626662343664316632663930313038643863 -65653630373437373631633032333663353866636630306235363035623837656661323735356361 -33343961373235613262613363636565396234613865303937323432656433343836373831343162 -37356234363662333764656530333139653434336137326631386366306162646433333239316133 -35303136633664336433386331316662313330643734383139633938636462346539623835653738 -63373439623866363863633963633265373339613130343966656637376632396139646266306666 -62316430343338626331326433366266396131666264633234323233646639643236323937396431 -62323738376664303237653563323164396166313266613937383431653766616234653762383163 -65666138353262663536623036623439363463306136363930343633383436333565363232376162 -35343035306230666432666131643634383263663838626466633133653830623330656164313465 -32363031313266396532313563343963613133396232353832636464663462636138623637646135 -64356364363862653465303837663261333933623266333736343938643065393031343535633762 -62363436323931373563323863643066653237393339653363656133316365346262306463336364 -66303732366437633461373065363730336235313732623131633366633834376563663734653063 -31656638623933376666393232613238323239623836313430356639613131303761346535336338 -65613363366331613865343163623863636666303336646636663465373065393731306634663239 -32653666343631666235383863336338326438646137656562646431623436323037313438343839 -34663062396564306138356465623032396239396562643933306365613033333065323630393665 -66346262303965353539663463626261633332373833346430346235376636323633623864353133 -34666265373936376339646363383834393166376434653233306632373966336232316339336465 -63313866653732386234666531663030313364626534343235333463376334663061316461613331 -64303962393930373036643561393762666334666538626233333937653038633734626430623837 -33303463386363386562333061353963303538366362313131656433383263666631666566633765 -65613238393736646637346333323933326664663862326563363566613361343539646264626561 -37323034323136376433343433316266386135333765643036373565326539643066336533343431 -34316264393261373130333930383932343864386534376633336239303066356666363438356632 -63313637616465386363346162653064613236383166353432636530336437376665643830356134 -35366239326637373366383962376165643932336632373330326533366535666434343161373264 -30323130626236356666663230613837373666666334646366616339313661613238666138353832 -37633066303139313961643935383632646662613766303366663335306263646439383534373639 -33643733626531376565623431666637323736393361373566626666633232383832656333333566 -65616662383561626161363061646334633033386534633533393032386532643939373736356532 -31633430613338343762656630303139316634323635653763386366633366366336383064613333 -32373164633066643064316535333364366461666332663165346462386563373864663565613736 -64636131346133623166656235306230353733373263313166383761303364366434656666303732 -34313234663139616466633562383663316630666261336463653236393832303363316438326635 -32343135396333613936383239633938393261393435653239643763373065623531336633313130 -64323761646666636464623734656236653439646433303934633536333862653930303133336264 -66333361323735386338643933336636303263613739666432626137393061343833636537396539 -62626534663636323764376437393237663839356534306438306431663937323364316431323264 -38353234303364653833393733383736636139336237386137313937656137636562303036666135 -62626530383037613439646261346130353430303131303936393230333662386533356631326533 -65643931313036623766316137353130363535383834323363623334376234393265326362653838 -34353665666537343232626137383166383761643232313539346461393937353737373935323462 -38303631383465623865323230333330383730333536346366616332343539313237333538333431 -65636235343938316635633732393430323932626339323236323663663932306463373039343363 -66363235366539386430333365613338646166313133346239346537313666396435343064643537 -30363236396563346165353032373463386630646134356634646465353136643635373164643263 -31396333353232323937346530303334623163316538353937396337636165356163396639373731 -64363036346434623366376261316632343238306564646234383433343137396462303466383635 -38353161383933336264393332386233643666656132633038323936353630313635313365353663 -30353734363435373265616333653737383838376463646466633732333537346565666433656137 -62333234653264626532323064386639643136336264346232663537393065386666623463336433 -38613966376535303961343361363930313664633030353538306633656335326430653534346161 -31613166366231386239366631623839316361303630663063336637303832373463613735356364 -39323238656232383632643630386261323764643861353063616130323939383732343863356633 -33303965363565313535386463613030373634653763313430643335393032663839636164303630 -38653930383661386331613763316365653538386266396430633638633638653134633365343937 -30313565663139366139623065623730356363313731663034663362313335346561323130396333 -37343165333434613235363964323564306330393738333635303763633466366664616138613862 -61623464656630343361353433313836316230653435356661326366366131386637363334326336 -34313334386464363261343831356163613834353639323562333930363563343963633833623832 -37323964336565383062373739346261306139376637306461336265316336666439353536313063 -31653432373235666235396239363261316333373332333234376635323261306234383836383161 -33303136363335353539663833306166643737333238643066313633323031613237666264666365 -63333766616631366639396235633466383062623462363638363866346462306535623839626231 -63666633393137316637616634653139316439626432366236663738316166643335333534626164 -62623861356134663637663665313934313833636165643732376438316563383539663037373432 -30326233373936346433323934663633363166376463623135386238393334396466316265613965 -33623362653436613835386535393038363064373233623266643466633732366632346330343438 -65383938363063666263616330356661343732313435383137313435656337353063613230656462 -63383465343932303832313335346230313164333362303935343836363764313933623831306130 -36333339373230386162323337386661303663616530383863333332396439313833303461643662 -37336231343065623635376564393934316333626566623765353265613737646131653934343631 -32613463636130333561653066343839303332396239656534303733643639343337363437656662 -63303732326235643830303038663131613330303863353231313561376231666135353464303864 -66633332613139386136303264366464356434633434656536393664313130646234393634326432 -63663938373932386564363834373134356464633662383130386337633564643930613734383631 -38643239663733393336643834643438303737636635396630333565656264653866366166623332 -35653430323365313834643862393065323531633230613630343738333934383161613264356265 -66626561386138396639353934653465373162323838303233636565306534633236343066623638 -38653132376663666136383235663337343762646137643935626365343162363439303061393563 -32633366623834613739303139333038303762316663663136656636663663386362613264663238 -35323038326239643062666336333932383065383461373933626339383434363736323536313533 -65363565306239363935666234653438393039303663623636376262666164393334363839393665 -62326533613639643938613838396565376334333035333962626135356265353639653633373438 -37316463303838633335306333346465386461373766356533313863333534326231353864303531 -32663461623731653138646336663266303065306562643961393936373937653664323032626339 -39373539363664316432333931633866646333346363336466393963656637366562336162613465 -32663230613335393839633633633063636635373363653234663564663264353036393434663963 -66653631663062666338326133636236303862356662666533326236613965323462303839623331 -35626238323837343738356264313266383135626239363731353639633837323132623635633435 -61386663366533326130626332613562633965633837346237633334303564363136633733323361 -32353361396166396334386630353738643438643537303034623830303939326436363239343664 -35633731396161623934633864313863323336326535313637653935376436363162363565666634 -33613435326539343865653862613263383236313130383234623766333637646464613365346532 -34653035393862643039623763323138333334623866316437613435666530396134306665316334 -31373364653865323765646535303166373638636134326538343563363732636337653931373234 -35303433656137333635383334376237616261393334623835353866323333393630373765613835 -33363636323664383136353863666635646563393764633531616135313935333539333664343664 -62633465356563353433633564303531633236656631326331323638643666643636396162316532 -36313839363164663336326639313066316331303365386130666662366638353834306662303336 -34393538346338376330333332383231333630353334646635313037313736613139383832663566 -30653137386339623136356663613431303564633966383633643830393636383734306238333831 -64363933386532346264666638316161393435653661356162306262323131643535636134396439 -64656538313938623938323638393734343464653362636532386663303762326234373930363866 -32306235363837396339383461396430373964396533616434373937386266396439303964356534 -65663262343461343864633934636536366335326162393830633232633832383166366430313934 -64643262313163323531326163323431386532386530326332326130353230653263616139303566 -64386535326262326463663734303061613435653964303764386537636132646430626530313331 -38323563303738353630653631333535376239393730633535353934316131333039353931616439 -35343531353332303932353636643763643035376161643536333365666466616234326237633430 -62666464613334633537306431396534306237313032306632323266613134363231313331313139 -63316661303830653035376261633539386533373235653839333562656135636433333263303030 -65333136653939616231363835616134373038633634656366363266386534343233393730386436 -61366536333730646562343333313234663037616461643864613334313464326661366133613966 -32343831376266376636646439323566373033643939663261356432623865313833316238363363 -34353765396233336263373161323433396437363463626464386463346330653861656162346639 -32653738353738396630383431346535616165306638666137616664336235623834616239613131 -33643866396530643062633537376565306538313437623462363963363364303738376662323237 -37313438633138666434646633333262643333633930396439333565663930613263363638366265 -61376165353939633165646330656631323765313430323630303335643362393763663134333333 -64383338343939616362363930626235303532666538653561656162396566383865363463373233 -62363363623938383234343639656631343831616564633136363332393131356531343137396534 -65626639613663323139393061353636313036396163646434613431323864303130663531343964 -35363339336436336532303763333564363962313637653438626235353339333438376433396565 -32646536383036353864356134623564653436326232643330613736386162656161336165363463 -37646138323937343839373736656332643430386630383463393637643463396231303730656530 -61633563633338663635353235663931303964333839663333323062646266646466356434343432 -38303762336132333132373532623637353731306161316564373935303366666530376639613166 -66383533663634333162373436313031643533336436373563356364323236333632303663393761 -32343631643566626433306532343764373263666537346538303561323465646237396639363065 -64646334633033653962303437396430646332306562623763323733303930623763393537646532 -38336439636134666264663339346230326435383365616433363535613235626439623037666234 -33393434306662333966316364303261643137356136326637346539363538353465313662393534 -34306132613636633936616339636632623733616432323564393565336633366464306661306130 -32373465656131346138343537373463663937636337303765366337376666346136643139356234 -37376436313834366538636136303965383330663437343363633861396634373764373863613734 -39363837613363663337323137616139633935643437353838383735383561346365333565316336 -37373231343665343838663235656132613331663638323331363763373539323239306232656338 -33643632333863333064666332663437613961393530616330643139663736313733306339346535 -31386639363338656462663566313836313066623661653139306662636331373038396636386362 -63623431336139316233396432396166633762363662616432616634363639616530663931366230 -37396434343039303232363633626532383235376538626438356639386661373330663838663134 -65666437363139663932383035303036343633303766343836363435313739363839373938643562 -33393733303165373364643961326634396265323962313433653634326436336564313338323631 -63666636646638626639323631323335653961363964616461393632316363623162653665333532 -34346532313436356235343539343330656235376261326262646364393866343562323663363232 -32636539343063323734663732303430306261366361383461363963346633316632623531363530 -34343633666136323238336530653335643730326439666637663266363563636466306531393431 -33306662653537643835656637636466383965356663373530386431383837373438373436383461 -66653735376535656637626231613437336663333132386135666432656638636138383939386665 -35346636313439613335653635653861303231663239613563333438303738356439343933326332 -35653764613061356465666361323134323331326137353863666364333332326133343039353164 -35663264326432366330353230643762623933346363623039656137356135613063366265626334 -64666539373230376335623365373866613761373637393066343536386134633131393339343439 -39353938643233636562353131336262616439623461363531383064616564616462383833363338 -31396436303330336532333661333263346233626532653437633766306230643032633035636132 -65343938643630303063336664393532363131333532323566306338343837323431373062393265 -31653964376339626632313734366665613563383266376533353664363764366639393563313164 -63646661373461323039636338313165306435323530633535353331343166636565373630333530 -63343262656538616232313633616166623534643834313138326634356334636635363935343030 -61373533373863633630616135626335376136336261646465666135383130633032656465623938 -32306433303937363133623537313535633164346536626465366663623430383639386662396665 -65393038383136306331323035373432666565373934653763336336323131353732356164366262 -64396137353838303934363033393363666632323137663762626163303963386461373166323734 -30336135396262323736313037653133623237336532663765323034383766363932666562343335 -36623731326232653363663937626136386139663630376564333230313632646334353664303137 -62363261316161356164313433666562303965646362626331323538663865386530393864373932 -63326161656561393665396461313938316430353131643035353933326430653439643634316435 -61626437376565363365303061346536386134356335646137626162663739656631343334653866 -31323564663165376266656233316266363232333336356137353665613961303065373735353735 -30663339343631633931376562316262356462656132666661383064333838323061616361383264 -64666664616231363661636337323839613830363366373530313963613132643836353062646561 -32396565313631623639666433663037336338313033373938653336366539653166376237326536 -35323134313962613962663630376663333334373431383731383038323965333736356462303566 -32633164663734366262383264376539313438343530303364623566383564376361343231386564 -39316366373964333837656536393961613466353436333839396331326463356636623738613563 -66333030376364383535633637383439383237636539623930623966356461366436323933646266 -63363264643238333432346432363339323461646533333266666630663161653031626361366364 -62613433333235356438326439306132376234326437316437373537346363303537626632373537 -33346534306363323133353163663932633632626561313566383865393461316163616532666463 -36333966626638336437666338356437643862303332353663303537613933643338396432663336 -65613565353536396132373663663730386664323732616533303037363362663335656236383838 -32666463323463363736316534393538373938303062396564333435383834646664346261383535 -65653931623934363330366361303030306132616637636366313333303664656363623061346638 -38626261313765303832333562383763363935656632343735613935356333346237313436326562 -65353230353537373737326239636465656566346635303134363739333139666362333964393465 -35663837633132663461383036353065656334303730636661363236646462353238343737633635 -38663762656234376233623264343762303434353366363262353232386131303231373832336431 -35636434346531376461393330363837633564613363373235623837373130336465383933353039 -31363732656164333037316438393532336435663035346139373439326565653363343336306638 -30383635656336643934623132383433333962323438303733396131333761643730343464633539 -32306664393738353334333465363966613261333532393130366437353433643439316135613861 -63303863313133333339323665343539343930356637356262316138613733613465343031376538 -30633537396262636262663033656430313831616635363239646334356334323731626530303034 -61343362383935343835316561333231323261643838616435313032313636613933363837343336 -34613131316239653861626434366338343232326531396530373165663931396136373536366339 -64306363393061333333663435666130366630633838333533306163313237653462306461303030 -61353830326337333563356165663062316165373939333938373637636164646536616639366634 -65353035353836356264346234653431383565373637383035323234316637356530346332643635 -37326161333838643166313664616331623532356235323564356261663734613464666133313230 -35393763353639323666313430666164323836633138373961666162316433396364393563626265 -64373565393965313134613365653062323764346432653630643662663930353738353038643661 -34376363633338353132363061363964346162663438626638373032326536363733396634303461 -39306432646362666336393437396330643864353437376235313966393837393435373437313065 -33303439343736626436613663353739656264313639626633323265643966643566623765653739 -31336564393138356265623636306266363861323961636131336331653632626439613461383735 -61616265646365636133393031333733363537366262326632363962333433656366313965303164 -66663335636363343739313838633235323561343833346332353532663065363439303936303266 -34356331633961383264393765316236376130323233346163306536386436366636636139363035 -34623938366266343164633532646161633264316334313930383565623766396165663664393537 -32373266656134333136393965366538316366313735653535626635663462623036323832373964 -61343236616264373563386265656364333230396462653833373338356233393962663965346636 -39313261353263373364373034653338383632643430323335306632353331633939383137396164 -65313166633632366465663466633733663939303039316265313539303564613038313136623861 -37383837653262633365333063646331663562356162333937383538623662633134613364653033 -32613537633639656361336236323039383533303661366432376665323432386464393534633066 -32616666613938373066313866306330393061396263386563306661643831303266633465396364 -30613961636333373462386531373264386332303063663264393861363263666638643836396234 -66613235643365316233353262663739313831373561633339396165386530313538653538633039 -36353364656136306161636135663464343033306161653365653166643762343035336166316164 -66346363646238636438353036353631663135356538663762323030353965386635306630643131 -36666330663063626334386431663635646561643464616636633735626132383961663265626631 -61653365373863313832366538363864333537643336623633343334633333323561373638613961 -34396433366665333531373031336362306364633537646565656630323334643038383535383036 -36633431633930313065663337366533323463333162326435626264393162333162656236306536 -62323032616162313666303265636636323163303234656230313061363862343736373630626435 -37616662393737656132326537336534623633613133343833313135336330303536393364336363 -34383136376363306264306237653236326166323032383939616563353432323565666263646164 -38346663373335636131366236313134623435633766653266383661363532353931323831363065 -34663732643435623436323236316365313861346162613636323138303831353062616263626436 -37356230643833323062336465306366613637633834303762323838656132333130646462653766 -32393532636463363634636665643439366531303162306133626138653839316433343532646530 -36383066303539386136643436393764613937313933396335376262323064663363396133373739 -66643332333135333437623266626563356337366665336632333035323537343762376132373331 -66303630353430313961356234616438363436323938656466636264636364313431396165623530 -33306334656338623237313333386135313961633666363031376334653433366330303262363832 -32333139643434623031653137393134343961353261366464383537643237623166396431356430 -64356435356566336461663766393537303737393139653132363737343238366361316463306136 -38313365333863303734336162666130393531356531623865356431653237343432613238633566 -35323166623935336361653530373466653063383339396466356334663066393539356432616362 -65643536316137326533363930333966666632393334333536326431633962353362313532383033 -63376566643166353136353733353862643033363035353534643062303133633064366332623938 -33363862306263663737616364633462363736666530613562643865393466656131663663663166 -33313839623739386565323133323263613939623838396365313265363466666235316263313535 -64393936396332376635343362396564346634336634336663383566376365353836383739333665 -30653038346161623536633361363961623931326265666566316565636662633032663639373361 -62653665366338656532373734663138353238356535353036336331383635306236656130613633 -34323962316636333664616664313163643061623836316432366664623164303365613161653466 -39333538343435383066343336313564386136363363326231353838373437643136616336393561 -62626430393735326366353839393166313030376435303039643739373337653635316466643236 -64646237303064306663396439316431393931626431616630396538626639636432326561653562 -61363338623863363731356566306231366336363739623735313134613933373238303765386234 -61363034313139626636643533643738663266343235333465663632343533646131393130666632 -31366562643166616430613030333832303636616231646239646435616530343032376663653031 -65636561353734656562626463643361633362313861373233346633636438653335323761373165 -31613130306437346630313466393735643434376133336565656566323132653962636630313431 -30386136643462306261383135336362356161666435323132626637346365396436653835363733 -33393732633464626437366439303931623962313433626533363136653939316633326432356565 -62393332656238303763363732623163653663373536393265343763373365316337323564353064 -31616634653837396438303938343562613761363766616561643032643065346238383734376433 -34613931366331366138346636376566646539303430353665626263643465313133323939613361 -30613063656665666439646362656136656439643934623730383736353031363633653036643161 -39366163663035663135343462336564383862336531656138313431316263653064306334633062 -39386538616361623265663536636130383362633934313764373538643636353237323133303266 -65613765313838363236656334663635636264356331666134643063636334633131353734623632 -38316637306163363839366536333365663337383632376230373137386535303666343362316663 -62663333313634313361373463313566363063363665323565633935303738316135393264663935 -63643662333430633535663130343930373763366430306337656162383537376564653439343430 -35656430366365636165386333663566313034646564636261333361373030663633373861323065 -33363463303135336436613563633135353531623339616530353538356130656263383265353735 -66383964313665623133303130323335393339663331323431363438613637636665623634313862 -66336533626266343831363732613331653431306462313830343830336339663636396566343134 -34396137396461376433653962613361323334393766303330633237623963653033363532613761 -37316139336234366633343064313965393965353461383433366630633935336233383962616662 -30356234663661343232633435346532313737376234623336366161303937623964656331633935 -62333537303536333062613132646237373031663766623730666330666364313634366362363437 -34643634303366633236363665643934323864303732613461326238323637393733393964663562 -30386664636139313335333238373632353936646432326436666665326365646565383766316362 -62376463626338643937663163336166663465353936626332303739393762313235323535366636 -63666438636230383031333838613333633734623134363833383035623564613637633562323263 -64356662376532333261316537656234636338393738613730656530613232373662363431393630 -32323230346332613038373530643563353162366366636338623566373065373637623638333533 -34633333663832393438616266653563623338626431636634303234333636613033333831316264 -36333664366262343261373163373763373036653866663539343337323761633431313435323465 -35386539636338613666393064303363303263323565613837623866633031303834363837626531 -31363838326235643163353731333938386264323730323036333066326463326537613530373832 -65303663613361336565643366316238336135396334303432393936336635306434653930643631 -65393438643630313330383061643031343663393265646664356636313035313961383634316466 -36633139373564623466386264303664373763386534326234323337633732633232346136366133 -61363939336531633265396136623031396534363031393164646661383339653232393463636439 -63306438613837313136646232333434323430643130313937376230613531383461306262366636 -61633137643164653739343663393335373065386339393531343930373161376434356666306132 -35383763613966353662333362396265326532303238656133316332363335346437306164653230 -38333263323064653533373365336166613431303231373062376663343261663135623937373163 -63613239343034303263306138376430613335643862303633343637373430336131623037373537 -64323838306233396435656339306132623266363961643166303733373537333135303665336239 -62623439623865363736343631613432633264363733353834386335333330313933383433346137 -32386432393765356133353736306430376665626665663964313037323136326164653435383332 -61346361313063333137356662383533623734633435646638363537303037393638316439316561 -62376361363532623266383635323330633033333536636262386239353732633237663139393839 -33633537353136376163613334393430663366636533616265633133373563653962393238373862 -61376639663631323231376631353964643162616538353835363937663661313632333333373933 -62323539643630303062333063386163656530623061613664376436623732386666336162396533 -36343563643032356261363234663264303762663439363664353966313561623761336138313165 -37633630663131313565346539396234363565646530386266666232306262663961663636326536 -64653638643935636365313561313964653764363766623433353365376161623333356164346232 -36646630666333383234343931343161646463633134316466616436336634336661356161303764 -62646362333139373761333035386666663561643837623665656630393432303839666666303339 -31646461613838353033343665396239616538356164333436353133653134313738343530313366 -33646530373561633834383865393961353032656339633732646533363535343630613937323461 -65363262643937373638633838343266323966623039616333656538363362333962353066323363 -36326138323132333837343130363735623861616334343130363064303030306361336161303832 -32343534623662393566653131363866343637373930643434326338333362646233383231636235 -33636433346464363766376634636533366163333064313534313831636238376639393533616566 -66333633613261336361366234643966663536653736323235663030626234386166663033376234 -32623435383837336330616235313165636537623462626230346535663662386337366532363733 -39306631626262623630393137653763326234366139626461643931613736653932316466396163 -66643038343336326533366438356562333561616138383330643235663633336262303266613630 -61346237313730303666346639366439313865663430643536336338313130656438613831383633 -30613862363961373732333863636134623431626361396562393437613533383738613761363835 -30363831363963623133653038383031323364346237326164313232363936656132353732373463 -39383737303034346163623737353537646433366539663963313566363032343836383134636131 -66363664336435376563343765623737366632383431656166303639366661633565623331613530 -39396131613161663436333736666536656364663361626633333637316130646266346165323632 -35623062353732616535373166336564383166663861613239373836646637616333613639613862 -66396434333039393135623733653938336233613235363134636362666363303337383335303136 -66313061323938643432623833363930303236633462363663323362626332363565656135363464 -63376266653132333565616363383564353135656431393062393333626336663738366331383737 -30626239366666366263353662333166303063386364366530653731353135313761343461646238 -39353531656232616530633330393130663862623964363565326233643431343432313039333234 -63336232336535373032383933393763623538353061396534373632323165666134333465303066 -39333366303231363863333133643062333230316361623635656663323737643965663563376564 -34343134306333386663356339616461646639303835333333623363383364666436643436343533 -38373364363765343265663138616439316633623865343939353465353466613065653366336230 -39636664303335316538326136356361633366373761333635613139633665616433386162623965 -32636266303031336162626132363931363163313235363865353334616137643934353637643633 -33626137376132353735303737343433323462343830633731396565383037313232313862396330 -65383564386564353634636430393562623833373533626632333530336266633038663333663063 -32313534666162666134373533313730663730333933613762303238323065363236363166326361 -39633938373861366133316261303665356661393830366131343138623138616233623961386233 -65356634333833313061313134333364306238623637333733666366373762383732663037643338 -33333263356262633366346130386337396263646333636134333935343835386366306663663962 -62306232653735656365643533653635366232326638386563393365376464653532656136656266 -62393830656339393033343638643334313765383766336338363331386135373864383939643166 -66316437666434643266326235663632363161663338336434313134643533383634633936336665 -62653638346633336331333364623363376538373562663437373933323932663533333733633036 -39386333653636646133366461633237656166396664623136633863653339633463346135393566 -39316161613532386432306364333662313261346631636430363061376332633431356132663066 -65643137303961383539663733653863346131396232346263393934326239383735353565376533 -33356438336463323531326561326264396635623734663835363039613162613762393662363562 -32646634643934393263616335633933386564373162303766376661323864306230376533633539 -38363032353639633735356266616330396333373665373035643334336566636534383831323862 -33623866386539376131393531653562643430663765316539613438663733366566366533393761 -31303836636533656535383165323232353065663338646337323530363133393263653930666334 -62323836353563366637383138383864323438353339653434306532336164306238633437646235 -64366536393039323339616130663939393562363935633435373932396462323533363838356663 -34343138353130383734363234366661386638653834643661353031353031343562613461393966 -31386463366635353965386462653461653866393864613063636564313638623734663864333334 -36386239336431316338643761363566636137333161663833623633376262623830383961613931 -33666136653631663131626631643266656430356364343138353663326435656663373035333934 -33636339386639666532333763393735386231376666323733333030353239373333383264323130 -35303437616433393231653739653230326263616233373166316237346138336364353339653339 -33353132636262323030643465313231393763613861343734363031386563646337336562336463 -37303232663235313335336435333032626432633531306638313561646236616230306135313639 -63633938326266656138383635653339393238303964386663366638643734633866333432373737 -34636433613566366132613864643135356136336162643066633035663533666237376437333762 -64376262633064333635623438353732376466366338323636643662323234326139316534646533 -33666432303636373963666334376263636634323532643837316361376338616430346464656465 -33373336653539303163336239613038386230363839353532336335303564306363666665333530 -63393863363232653138356234323161323931353932326137653430363836323863393035356230 -61633161623761303335646435383934663135633734346433643639383932643662393264626430 -38303735343033303765316365353834633461386636653662393538643339346230643335656438 -66386164326535646335356661383735366134613939636130383034633734313739333635376233 -35623130353239323237386337623962616536643463376632613631646436353630366137343838 -30376136616261303235643437613864613031346564363465646533363033303566613061363330 -63306530323938383737313438333139363862366331336665623139626135353236316664643661 -38343165383537643136623661663730383635376263346131303332646464376263303464373734 -62366331316166633631663163306237343862306365326339653363336638633830646233363665 -61393461326535653736363834346163343736666665623838643937306539376266393761633263 -39316163356262633137313261353463373166633735663035663135326437383534616239343661 -64633762613833653365353234363539643533353239316238636537323064353337316361353630 -39343463643236656431623861333734313838346430356630656131383763386534373635666331 -34333137346266353265376431623935336366623461666237666334616634613162623761346161 -35366334313837333939306563316236313839623236373234623635656138656165313662656632 -31343961363864346438333932656264363932646132366136663339363930386435666639663135 -61313037303762626537393330366131616532646666613338393032353837643835353436383539 -30323064656161636534653664336232613565626162383035636131393539386633643436653633 -37343339663531396533323432643464393431623338343537646138653236366331366261663631 -65386666313531323031353762363437653561653534316463666539636439383361373861646331 -66626434326566653534393964643938393463303930613734336161356431363230363261383139 -38356663323539636533663237306566623062333036666636663637306332626139333738386365 -38363635386439646363626631366430393636613763646539303939643764643966353461343466 -39363331343334653637656561383638626637646230346665393366313366633366653839663839 -39333263313566626135326633653238626566633536353461376462613462303830353638393161 -36366433656261313436663466366433663138613561386336383262646663313836633564353266 -61623139323961346564623939303339366233643231373031626165646364376138383935626166 -38373833613033633630313965336438626363353031346630376532393932353961363732623631 -35613038366664663337653230653134373861666430343666623165306130306137383163396235 -66633035353534386262663336646361303236623365613736383938333261653462386636376531 -37616539376637386462373063316535636365626634613462623232333435313733326261373230 -66373061336231663230386133613665396263646563376534313830316439623363653165613666 -62373063633863656566643333636461353339396635396331633734623266306430 +31393939323063366635653130356131666436323762303533333634333166396337383133616565 +6134333735373036313732303663376130653536633031320a306634646132386335633431336263 +32346162326338643665386432393464303165613930306636336136616239633137336563356564 +6536633364623763370a636563646435363930343838383166336438656365373535326666386432 +30353534376235626238616662356634653132396138333163343736333631636466343035393337 +65613837643733353664633262323262616331636533306561623364303437613136336238373938 +64353862616166303339613430646534356237623733613035303066623437666261306130326366 +66323331336636316566363532316463346162313239646431613865646462383432316363393661 +38373837353764633435303335373639643431366137373930373464336463376136306263636163 +61616632326530643531383264323565646466633963663732653561333430343162393035386330 +36323566666264646631323939333461656237643736656430346434613566636663663165343665 +35353239303433376635646237653739396533626538633332326462393838356262646430383138 +38633266623464643463326563613232333361336162356561623265643130346537326238386134 +31653061376334663534643130323032626232636565396466643664326263643730306439663337 +63323630363863356561316139343766376132343832663266343232386133383135353861363231 +63643763356361396166643262353866613235383033353562363761363539313334316535653462 +32613139666234646238613837636462653532613631333332386339636334616535326333626431 +61616432336135363536316137616364306337623966646366376233633437363631306662383565 +66626433316135353339643235633933626563363334356234313333356237666665393931373837 +66353535343339343631333662383238343930633630333133656236616335663439393865356564 +36633334643739333035646139666630396239626230613264353237383830653461613634303532 +36303730643632393731373037306261613037373630643137646166316136376633353865396335 +34663233633431323233633038343961626538626161336533316339343539643764643531663666 +34353234346362346263656165393538346435663465386164336465313739656334383339633661 +31613562653062623064353666666564326339626463323435653763366330393463363935313834 +38646534343261636362633464373034373432643438346364306163656164663762373133366239 +35393231323337663633613936653534356361623865636535346263396665373266616231326362 +39393061653563333562313639303031366236336464643864613939373265353961363739353130 +65323761346363336663353763633833373633666661353032363961393834366631616439316161 +66316438316135363038333831303563363137336439313933393333626261616535373932316231 +32333131356330383239336432663761626265313266636233373363343031343339396362316535 +66373234666536316533616635663837363361313862386364343932336430373934613934383062 +64393835303235303835363633393430386439313536653835373164663064383936363335393535 +32346265346337643865303030393234306565386137326637353430353633643939666639636162 +33616537333266396535643536346530613637663464626238666464346661653235623064323763 +37303839643137376438376237323162663636613431306130373562323739343636303936623434 +39643764363065363231613065366232343638343461326635303862663834386633376262613862 +37623662346139623734303239346366633231623937666330326631313637326466363833613534 +36636639323563386161313038313835653061613530343538643665663439383361633365656662 +62643732633634626139643630386531303634393432636235386333653732336234663738346562 +32363832326365383137333566623635376335666134396362343064346131643331666666343463 +38633837623337333061373164393330303161373632386139656336323837383637666138663832 +38323763343237646138636561363436613063323962393362303630623036353537353061643463 +37313863623062653036663733633731623831626130373265326231346363316538643562313036 +37623566643636633938353235653364616266653861613362383464346237313031663063646466 +31643463363138343037646164316433646232616430353338656237623436376534366635306663 +61333166363537386235663563323834336130396166393133313036653631383035336465333435 +62336433613235616234376133366563383630613161383830396165646134323139313437613236 +38656531333137653966326462613534373830616664326362313064613434363338343564616230 +35316461656135336563366637323933363763383165393535383566306332366530623735383034 +39663434336266386535616430656566353137313231353839653565663430363638336530613462 +33326232613964636231386237643063633335373234346165333336393234666233623038363165 +65336139343634303462313335636232363637356437313161393739313966616364333335616261 +30366536623663306535656431383534393530666461626562353262396562323638663036393934 +32313361333439376563353130653537346438363830346138643438613433323731663631363732 +63613738363637613230376337373137656363393261326461313233343466393766623433383338 +33333533623838353630366566636661353730313737366333653537623637636138616233316630 +31363362646565383534343535393266396465323634653262653931623931613565313932363961 +36623161373665393466306161323862303830616333316563653165393736343638613434323766 +30353331633039393461393062323335316662366537333035666536313036346266313839613031 +38396232666165633437343032663565386133633631643863653539396361623232643532393665 +66646334386565383338396461326366393833633037343238306635383663306635356465613932 +61326663363634373737336265643836396238373761356665316464626138643063613632663461 +65646366336633383765373034373834313731356330666363363535343934663366326465616130 +64323533366235313064306536306238666366613839653730373864363861373231633164313034 +37313361613666343531663965366239363735643465323763373362623961303063303536663264 +63366638313332356164616435346335356536336266646263346538393136323537663464643265 +64313730396638353465626330343836313863636666386334313432666230613462363137303732 +64316234306565396464633662653831393934356637613139633362666661383530363032656465 +62366639383433393861316132323634373366376539636462353663303466383233393830646338 +62383837633533326665626161666530346336616436646535663930386462313165653436306633 +34636238306636373136303866353832303265613831316236653832396466643934356464613331 +36653466363561373065363934653434616661323438353166383564343731316662626264313034 +63626638663333303938613636666239316635643931626538633465613465663235383137633839 +61633761336665636164616162313838333937353761363635623033363964616137643035333735 +61656639656263373535356331373633376463326134363031393765623332326136343163346533 +35373732636131303561393462656637386631343736336636323139326234356238336238633637 +61306462373633386136636636383137356437653565346130316262373436663838333065626239 +31643264346363346237643064646263663931353663613036346663623031623064363637356537 +30626336316531313339363937313938393864633733303738393766346537636232326232313833 +63373563613935653965653737383438643566623532303939336231333639623834323639316466 +38653635643032643431326462383634343235663630316264356431396561363932636136656563 +37373466356538366636396130306135636637386632393763333365626339643735616332393236 +65303032343831363332646564633861383865663731343632303263636136353962313232333938 +61333633633935623365393032656133323434626161633438373766333064633161393434366235 +66303566383132313538303637623836623163376665383933336533666530613738303262633333 +35356164653063633039363039663934306333313163363034656463646464656436653438323362 +39383638373337306535313739376230316166386537383736386163663861653939633338656538 +38646431623831353537393835313130623336636535313765633830303164353932336632373131 +36353566373662663437613766623434636165616562353938336631666665343661656363313538 +35376633373535616163623838616561363932643366363235383335373632626131313063646265 +63653236326135616361383339623663373331343735356334363761333766376333346332663733 +61643231643465663933623735336532326431623764396136633863353135633432356362633364 +37633138306266653339643763326330373637663831366465383035613463336336326533333038 +36643961656336626537333161656264383334323063643462336331653663333636376663653065 +66613231643363653163633330373039616138306261393734303536333763313530373738313266 +64356332396137383966393238313033393934306264373234633132613231393132336665663930 +35356364666334313937323931636433636164383562396433316164316232363262363132653435 +66656239613163353866306631323961623731623932383265306339366134346666653664343639 +37623766386538323763353531613530313665326661356566373035323833333033393364343439 +64343133366334346532316439316433346238396639623633373534323338356263643263386332 +39346339346631333733613064393964326234383434626462316330633031346262616338643737 +33353866666532333233313362653765393664323937666435393435373638643039306366633539 +36336132383032623864376563623634623461303564326639366361353961366265336563356436 +38616538303831653533323332363863343666383537633864356665613139356535666136623364 +34646563303636643133386537313038313066303735383437363531333739353166333635333762 +37356532376132393862666236303332666262326135663735643663346234376334643966323339 +35626437376162366638373235366366363166666438363964346361336361623234353337623062 +39363234663765343866353964393438316361343532623766393964373838323363333633323134 +37356263303564303534656338323537303965333138326636303138636534323461633837646462 +62373839396138623739346466383830346535663534313265393761376135653465666630626232 +39316337306161356239633435663166336139626232653261323463386363616636353138633133 +33303134646130643261333630643038343637636132323930363638623266376263653561306634 +66386363656233386639356462643234353464323239336464646236383537306539356662373865 +32653865383932636230656662333530373663643363656166636535663862616330386163613337 +39623864633435313735386163666164646335353562306162346339326231366566613735353033 +32373762313636343761313235306234343639613463616263386130636165393636383662643739 +65303537393437373866316535383066363732366130646533663062623361623565653138383433 +35313335346236353363323434653434333432316531393433306435336238666561613161643531 +64393732333932623266643938396132393235333030373133386638643831633138613835366530 +64633933353465646461356465633239316631666264353530363938343732656233323339633061 +63636364323762633639306336613532333935633132633239333161343933383030663539623732 +31616365323265373461343961313436613436633061356630656433646361373464373962303534 +63643039313565316461386230333964366537346162303838633133323339323636356430373264 +63333364383363613637333463383238303734323130623437333262613763663535303563376631 +35616133366439313362653763643135383061613362306533353638633666396266636337323031 +32653435653639663335333137373964393431633261373436653036393366633639646639643734 +35343639616637653464663964383961636565613363366561353838663464646636393164626162 +63613763386262633936646466316531353164323036613539343938633830626361376135666337 +30373038656339373665613833383439363131323562343435656633353266323762323764633336 +66313561333066626634643531613930316664373434363365626337393833393835656262343662 +66306138383962326366353832306164663561353661323538653861353338633935636630346135 +65343964323839616334363430323563343463333832623766633364616136393934666238303130 +30343662303736626639613034316336313133633630633638356339323831386238373832666232 +33396162666134366439626564333634313666376138323039376636346436343031633361663130 +66646364646235353531336132636339613138336332616165376138313764346630333233613332 +65363432393532646663333465363231363535343961326631393565336363333362613730353733 +39343862636131346438386165396334313230663461336264396666323962643561653139323664 +66323436663539613832346265633962616531363731376261316463373035626434313731623864 +30366561333161303934386562643835373336313530633436326334626537323365663138663366 +33626534363365376633626564326266666431626364623832373865343035393963383838346362 +66353835393439383966376365393966636131306464616636643035383466303064306635343036 +64653738303232356437623937343963313837653931336339396561303864393739356135663664 +36656163653266613330396236633566366331363236643861373338336562323066653766346232 +62386339343161383732353838353764326264376432313865313534343734326538353364623535 +36376535623834303038396139343739353131376338363738646564363931653634323639353331 +37393237623864316433313139376538356538393966663065616131613931616638633536613930 +63613635643565386464303065623966623038376338636663303435313764393838626563313335 +65353636356330613132333338373764353766353363333838333839633635643935376462393030 +30306533656139666534666162366465306163393063633632303330356165643135656433313336 +62653237393834356334613038343736336162343337343666333935653635623834393139313230 +35376233653937326630393939373365643338353765356239646262633632643161626161366465 +64303630333433303631396135343931306363393864663162663538313263313333356262326464 +36383962616163356166636137323230623262343636336364376366323238386663303336653738 +61393138396333623738333766636563623834353737333134303733336165303235303635373361 +31633035626334313336373733646364643333373063613063636434383037323931383466633164 +32396233633233646630333037393635663733373664626665653533303535633034636163643937 +37613537616137303261366533656536376437353839303439373939373764323563313363396134 +31326632663664626364373133346363623239313166653737646335633833393838356131666233 +65613662303338306638623462373961633338346130333463633465373739323436306232333731 +61666237326631323434633063373338663961663837653038326233636666336662303737363935 +38613663363430323135346566306664633930656434653833323539323266373931656131623933 +64613763643664633263353162366231646262643164653437316336343438386665633562366334 +39383261383265323239306266323839326566303666623462666236616266626439356266303134 +61306530326265623132656662653335393962396135356561396231393831653331313765306161 +34353064663032343438316163633333366266363839653662326238336631623863663937316266 +63303936663666326539633138616263326266613534303162346531396333313930366466373161 +62363064653737383264323235393935656534313830653137383834323363663237356230383832 +65363139633961663063633936323339363839663361383361613864343035333135633038626234 +65633662343264616637386463656261316631303536386236393635636430383563383232373938 +34656437343633333438666461343665383066626230376233373236663561666135663433623137 +30343639306333663530646635366166626432336530373363333131383338363966303639363765 +62623537343732396264393933396366376634626239363761663366636433336630393036343162 +64653339633131363035666532613433613735313835616433316665386133623762396363626136 +64653333613561613362333531343961343237383564386239376438613935393066383138613637 +38356232396432623063373138383231383565633264356562303031663538616333363038363264 +32303030633037353130326365386332623634393139343061653136326133653632613330373836 +61363166393766373232633637373265393539316134303961336561373534343137346434663462 +31663232363638373939626432316232646338633863323031646635623735303762313030626664 +34323361333431646636386132363237366636633332343434323765323235316134386636616231 +31333564383739663537663131623066313336333630316336303130326531643539346236326131 +39393034326661653763343761653162666561656564356232383436323864623838363730616566 +34366464353765313663646430656539663764363562356336373634646232383338343737326662 +38393964643261376562336432393863643733656438313462383161613730656235666333643233 +31333633346333663563363730376164363363356264396334393065626165353834623139396235 +32353734316137366131383935313136313337346436663735353331393030366330623937383132 +33366132653534306661633765393833656534316533373163323333613839616165356636373131 +33656534613830656532333533303730323539326439663665363061636431303431373662346462 +36323265343939643936393066303065633637313437613832303062366334343139643864313264 +63653665316638376433373861663630316439636130626333336361316165383535316466626565 +30306662666537623963646432323262386132386130303335653634643436343334333130306261 +63343666363536616236346637633632353839356636623066386238353833623565363932313766 +61653161353835373762333034613563663633353634663562306634373938366130613831336339 +61656265363662333536303633373233623436616238383463333764643362646532393630646664 +34653765613362376238363163326433623963653334306231343135636230396562303662383738 +66366531656562333533653638316336323365343063366332666436366161343730313433633362 +61393461396162316231613265393932343866663537663333623035626635316466326664666237 +37386661346634323438363938656339636434383931613931613262386532646466393333616637 +35336563633164323535326530393134383634363461313366626139616433393935376164646661 +30376638373565633533343730623665303438363361363065373166383662616134653066333132 +65636233373934393336383631313932326266366362393036383434633333333333613937636464 +32316462373532313934303961616138303862386537333439313065393630386637303161393330 +39346137326338386634313838306462616263353065616639323532336531383238666662313664 +38326163653663353332346631376461306663353631363661346337663266323166643062633132 +30393863303130643534313035353631636631336465613738663361626630656561386236326661 +38643630393963333036393534373839616236646466646433313034353163376634393038643838 +65356636356265373339393765613164343834383762353234373831613264356134643739666337 +36333162353239313437643336636438656266646536663939333765633862616530386538643935 +61323864323632626331383461613534336534666533376566323936363031343537363432396431 +61643339643038353432366261663633623631396463373735626433623032626333653939323736 +64633336616536393736636361636238393439386165623039373537326462633332343039303339 +37376339623837323461353739346266623334363439656133326130336563396338363963306433 +61656531636232326539336566616536613836353633316666643633653131343161626332346334 +65363136393539393039653134353734333964613533323061643563373332303935653837356264 +38396136313265333063643937663461633438333039303234653662616666313732393038303937 +37336133333030643133333539613538373461356131653262306630323734663532333932383163 +39643838366263613062316331373537616137343035666130333263663361333730306233663432 +62313539663138326239353932336364383337636564616466633830353030323664303961383865 +37366635653939626238376238613730663333343664393966663461633866366632636630333139 +65636361363339396337663937656533393639363336333930643530346361313832663864633239 +37643439306631353263306664393938633139653633386466306563663333396663393266616438 +31663963626564626461643038653363386636646537626561633464396535323666336264336130 +63653238366565336431663365373436323938363764366137353034663666383537346264343136 +30613734386432356536353431653562363738363762333632626630303738366437313431656365 +66653830663236323865303131373335376630333234646232353731633764643935653631613363 +62343063623133333162343066366232303233353565373739396466376239373163393065323132 +32643636396532363531356530336563626164383164306161633166353262633433393731346631 +31383730316130303137643438316332353132666436333232646435313161663333396537366163 +36633063383238353665613333643262383365396439366265653739386264353436653137343239 +61316465313835643565343733643330326236303463633032303133343662663834653462343837 +62333334613066383537623838313565353430323737376237666164326537656430373032633637 +32626264353461376536376137653433613035346264363332393462353562653537626262656663 +65366237373363383738666134623638613138613962613936353933313061376162396532653635 +31663932343936313831313636316234616130336361653533666431313334323235326565363131 +33343630643764616230376434626234343463393634633534663361373465396566313862633834 +39353964376434633130656535386239373963653832643838313833623839353866386531323062 +63633665383935663965303830623563613734313034653139356435663733393836353834373435 +32333636363937303039323065653134623438383730326262313065633265363866363435623336 +33623536623432376365663264386539303363663463313035356164323038353264363862353964 +38316235373434393964616233346430356463383230393730343437666437343663393263376262 +32393861363264633130656266633261623130613362663537373332626230316161646539323264 +66613533653633633262343134326434336166333634323034623738383539373230616237363565 +65393765613134393465376435626263323633396238663630373564336136386366326631373530 +61633738376236343863363031343835303931363936343364656136376330303566633962323232 +37313266393463373065636466363665383665663362303137613138643530393137613761653633 +36373363336430336366363136323030386537343962623361373265393132636130373464663335 +33396437663064363634363132363038663438396233303364646266393931383862306464366336 +66616131643662393862303166326165396135323765616366333330316563396161366561363132 +63373635613663366232363238386331316461616636663439386434666165306330366337356534 +65643662336138303335623765386465643561613039663536363734616264333431643337323437 +35666533646432336165343761303137633130303164386166356337656337356437613862336132 +30636235643532356236623762363239313163613432636466346538343062316435643735393331 +66623331626566303535623631366135316238346165386430363062383138373132656633633939 +38643765323636393932646363313265333637643535333762633939336162623435346664383231 +35346664393165643239366436346233343566646438363533313964323831303566666264356337 +66663638633064643962346136616333303064356331343031623863353365613661353333656362 +37393062356661383430353365643964343462666436643236386164313932656234383039633537 +30653833626331373930666239623639306433633038643963656634616337366230623265626539 +66323638616330393135356134373735613933373735306330363437373237346532663239343636 +37393937646365343065613233333331353763656535373739663263636233363230636166363530 +38333065386138306165303664393064643836346264373939383535323365383363666436356639 +32323364643937343139633435623734303839383566656539376263356634653838616337386466 +66366632346331323039663863393666376637626137393037623161323931346565343935343863 +38643864643739356331316162633233666162373430623337383734336564633463643338666637 +35656265656461633537643739323833373838653130613831343336356331373737616465646564 +64353734333262303038663631663231326636633532343164643338333237333331316165623032 +34386137386534633432303638653366333962646532336130646139656136656533666533336233 +62646261636630663161313966323862363761333135663864646339393835316437613862383434 +63643436616432653662613531303361616263343365613437303838613332643637383439646664 +32333761666237306232633832363362626239326139363363333530666630383736623839303065 +33383562353733393830646661313632656361623037663262303665393532306335643465353436 +63313834333732366530373366666335353630663633373333343434626664383137646538336666 +33306666383838616435363037333062396461363634383236336130666635666539366462616238 +31316265303961323666633065633961313963663962383837666565366361333836656364346137 +65633030356432343935343139366138666137373734343361616132626630313833323461666265 +61383565666138393937396265306634323939366334663635663566343932656633366333386434 +38303530333034346139366432396663386631663931623761633961626539386234623837376262 +38326235316330643635383934303030663634633834613164623332393365333966316131316161 +35353965396337356463626661333232656236653930373531316163346465333862623734343438 +65643837613962656162343866336634353436346337643761336531346164353765323238636262 +35663432396165306464353266323532343363333232653865616563613030356236323036396266 +64376162323436646532626363363933386433333033363864356334323839613134613935383764 +30626435633531633361366332623861626363366531393861383134636230643032336663666131 +37643630313138623433316562643435323762653438333335323566353264323033373834316631 +66346332373366313234323237343931393437646130386261656565663065636132333035666164 +35663762373131643164323031653365333339326264626234623537326638663432666235343730 +61623961333439633265316431366434356161613166326162383732386166383036313838386539 +64636664366130306365333762646133336465313033306563353633626561356336313966383931 +32613863653662316136376330373335316437376233323930356434663732376631643963383938 +61346262333339613736386333323534396130313935663632663432623162643738313238613438 +34303163333731626664303337393965313237333936663664323565366434643436623539633139 +62363431656634663636646465396238323165376131396635663764356532656534373065663461 +34383765333933383561646430633235306533303134633939336561623265626431613833346161 +63616662616239393566396332313430393563313063383836623331393965623065643535333734 +63323431633533373632663939343830663530353533373730323939363931353438373338313466 +64326461393564383961326463326335376238353034656531643732633461353862636466646264 +64613635333334363237353235393539363766643838396330663437373639323432653165366339 +30353739383739633564313635653763346631343934636437346362373363316339396437323733 +62343733303834323736383834306232333932626166626536356338663862613339353531363935 +38653761663066363265383139373536353638623330333239636361646564303039343930623638 +30313864353833386263613633333335323932643462643133343461626365383238623639306631 +32333538613432393662383434393831306236656236303234656235353637393137326665306661 +38343961356631653631323261386630343263636537323434393736313333323334336639356263 +34653237313930636237396264653336396132343364616339393061306435663632636433373065 +37383139363666393634323636386166666331346564333336386463626433376137646561353730 +33633661616238623363366532336363386137333363346463303435353130643132313665643264 +37613430633733353837626238303833343463363662303831303734336165383432663562613433 +38613564643230326163333863633961323239376136323833326361653862643664666233306564 +39653738343431616563333563666132323639366461343039363436613236646131343332363737 +63633036646232646566623632383636303861646131373330623436333338346163316232363339 +31346163623032353861343234333337623536386366643037393265316533356261313136313237 +61663032363466626239336435303137636366626339623730653334313862306539633933336438 +65663436623033336263383438353838353037363266636438346266363363633630386438396464 +35346630336231393137366535333261313662663564653963316364653035613030323461643433 +33623934666562646537623936666361386633393364343363323532306134623461366334393439 +32643164306237393031663537636535396635623861383163343538633335383466643733653338 +38306161643436346230383763313138393435356435336466303831613261356336366234366632 +64323362373833336664323936323361616562653531373962396533623937386238333965333463 +63366633373732643939306462356163366464333364613738393732326130303262626537323966 +38313837643431393232633531663165373565343739363032336538363435613336616663613463 +30333739366165613165636561373965356363346231646462626235636431623233663038373761 +61363863316137393033306634343034666466346631393465646565646333343463363362363066 +38366139386634313533323932666335336131393766666538386632336264303531306264336431 +61303838616139376332613838376331336266316566613738643537393162623032346431666137 +64666339393332376636646533636531303266633530633436326563396462313838616232353162 +38363535396461326131323936336661656364663965313462326534323461313839313264333231 +34633163626336316666386133336133306666353632333033353533376462616265356438373733 +66666233333830393437333635326239373239303032633765303735646162386237306438646366 +65333534666637653165653362613130626432373731656636396662616533333635303964373263 +65646531633731323438636165303938356265383030653066393031383438303236323763623963 +33383130616364383438663133646233623864376131323765623261353834633331353265303731 +36306462396530356433356537393335323965663236646637306139363861353139626666333032 +34306438383737656136313539623837663232306137333465613938353161356530646666643539 +62333038306665383362393137613962323064653135376137626638613133363535646561353063 +39386165333663336439386132303334306330373730666536373236333438636663626530663239 +36663039616662636334663864323635636439616465623438393630383832353366323861613162 +35333666313236653433343836623931333765613837343835363762663563363266303561623735 +64353937623133306263393738303666323066353964373033383065316463383462326437653034 +35663831653336663231636165633433663530313138373035313462393531653339303365646233 +61303765396464653037306630623738306366383735386135316266356165376637396630643361 +36656237663565303433313332386534356534653463313363626264313132373266383662343233 +33326264636365303630653834646539653865663465326437623461363937653232386166373233 +66353164383130633832666438336261666537623164363263306433633337666335656134353765 +39373134623061653134636364643935373233396237386334653561313833343762323832313564 +37356238656430623763396439383965323136323066346636313465633336393931373931633131 +36363262336636316533396465613736663662306232323739363336666233386565373837326562 +65643661626264303434613837623761333934633438326666386538303264643334383833616631 +36646564383436346234303336666666313165396461373938316634323830383135343830666339 +63303737613439633335376163616432623366333933643135363664626437636433323731663166 +33303535643634633764643832633263376338313732376130336331653032663137613031333963 +39643561383633383932653066363461663331383962373631636632356336396134653330373961 +62643433363830306337663262383261386131626263303762663061353936623034316432613963 +65333738326264313235303730303165386463353965613264383631666463326434656235323630 +33656265313363306565626662303139353938306237346166663534353936633235383735656366 +39646330656533353030353537636362316261353966373739636336633334633262343336623762 +31383637376566616464316132346639626464366262343032363739336330616363373437346334 +37326133633234373231346632613463383530323661386364376634336135363438376239663263 +66363238643833333461626437356165346435386363356230346562376566386538633638376362 +39313834633264613839396233653162313735313230346666393161643966626230383066623863 +33613365323835343039336631336138656566306161643132333562373630383533613336373435 +37383532356336303738343565326265336631393336666136626334313539383730653462373963 +38346666663834333564646461356636336133316439303062643534303837633661303032393633 +35636231333561343761626431373966623966346466633563626538336430646236613466303331 +35363736366630363166396433653438643563326464636466633866353263633534313365353334 +37383963363035303735666330323435623531303366663866643334366461313336303334616337 +32343362336366623566356361646638633237306531396264643437356133653931396433383262 +34303039353836373738623339393136386538643738326264323334623962323838636464663832 +63656234636364386663333733663332393639613335666237653739316138383536366662393061 +38393137383331386362386565303135656231336163363839323332613530336435623365393361 +34343839373963666366633564343531326264323538393231373032663935343935396661313139 +61626662636235653335653761666137313430353533383033363762396636366463363635386437 +32383835653833643866633265666661363837396337616236306636333633616337336233353966 +39613232653635373534373039636437393630303063623033366563366664633233323565363732 +33336132653261643835336430613263323832333065663131343830316438306139653736326232 +65313764386231623434643738383536323232373463393239323865313061366364656536626261 +37353831366538653130356330303630313536336661353738303230336435333663393365656239 +31386561636437303432613765353932306635366234383631346261646461346338366165633033 +63363066363966373430376634633733323233313466303033663463323932376461366634323831 +64656431316533613930376337323736633361386565376238336165323364316264666235316636 +65666334363638336263626332346436366262626537356163356362666539616636623336343265 +64616265333138333334636633613466656434333961316539326137656565346430646564313030 +32653430303463613532313734666264336631333134626339613535346664306534343530343139 +66663737343038313563623039633261393364343235656563663366383039393933353262323462 +64376433386533353132313739393934306566613632613933343036633338353435353631643262 +61356534373562333466366163333338303637623732373063646465656536313435333034666536 +38303335376632616136363334356331313032333765626331663139643335383039333561663264 +61353735306131656635333062613365343131383531393838376531626631396233613130646236 +65396561633464626137313737323164646236303866366538616661323331653536613936353734 +35616661383063313837376433343836353064643937386138626636373263326662346663343138 +64363164646561613230323436373333623036316231643364366565313534383665666563306430 +32383539633661376366316261333730316531376133376166646136623737626664656461303336 +39653031636233646239626536306337623631353539373762663230393361313862633233633634 +30663561313761613064376232613166663433316635623134386262386231386262613161333864 +62626333343062363565663863383038363031333632336664323161383239306466643464623233 +38623034623031343130613139383631656231343134373039333237343065313139313766326465 +38383135306539363261386531363034653637663235353836343331376337356436393132633933 +36313166643338353963616466623764363166663239356139346333373761653636386435663363 +65303934393339353265613332366233373665376562383562663166303838363233393563353730 +34663937313864346230306330323230666234373364666135306461363265373031343865313339 +32356130613261323435656266323231633164376132353065336331663562666132396635653964 +61343633323532653238396131373466393561386434356536383134623534303333333336303738 +66353039643466353266366436663636363438383964353635663664316463373238326164333433 +38623335653366623961303161316261653038373837643733363534343663393065316130663035 +63623931383263316333333863623836623635353862396439623264613234306538633466366131 +61373237323337656162303366336363386139613635616135323035343764333166346337353034 +65626535623034343232333735326438303438646263303261326664656339633762326338336137 +39653730393164316530633635306333613333623064336635373732313431306631383737326338 +33303931626431643465316438666432613564616238343237383165366262303237613961383630 +36643439626633306263316437653866343362623536303032343137366535356165313430393264 +33316339323838663062623566646365663734663339653638636363363861313361326166663630 +34633433326633613035663639386434663430626464623761623931353639306232386533363538 +38666636613432303739313030353830343535636633353664666661643531356138613731363563 +32616239343063386536383462633332303136343333306131616563326438653136313830303038 +32613961366634303562653436626132643338336235383638653961303262383565626632376363 +65613865663638653739356365383731333664363732643834356339623138613838343631303433 +65326262373963646361306432303336376638383061303164643030393633333737363166366339 +34616637363261376462356230376166373630313862643732616431303161653965373032336366 +64656138336237386362643638316465373464353862666236336532343436633534363966333031 +39623933306631633366366138303237366638623435323766383562626233396436616261663738 +63363632323335336366343264396662383562366430323866653635663962393136373732646331 +33333963373261613161623733626138613037653136643735323331653338373830343633303263 +30356261323766333161633738376238336666353733383666626634313437326462383335616138 +656664303532663862373935343637336266 diff --git a/ansible/inventory.yml b/ansible/inventory.yml index a43d970..20a7e9e 100755 --- a/ansible/inventory.yml +++ b/ansible/inventory.yml @@ -93,6 +93,15 @@ paperclip_server: ansible_user: wed ansible_become: true +multica_agent_hosts: + hosts: + haunted-mansion: + ansible_host: haunted-mansion.magic-kingdom.wdw-labs.cloud + multica_device_name: haunted-mansion + spaceship-earth: + ansible_host: spaceship-earth.magic-kingdom.wdw-labs.cloud + multica_device_name: spaceship-earth + multica: hosts: muppet-vision: diff --git a/ansible/playbooks/provision-agent-host.yml b/ansible/playbooks/provision-agent-host.yml new file mode 100644 index 0000000..a51aab7 --- /dev/null +++ b/ansible/playbooks/provision-agent-host.yml @@ -0,0 +1,46 @@ +--- +# Example playbook. The role takes multica_pat as a VALUE — pick ONE of the +# lookups below (or any other) and delete the rest. Changing secret provider +# later is a change to this file only; the role is untouched. +- name: Provision Multica agent hosts + hosts: multica_agent_hosts + become: true + + vars: + multica_server_url: "https://multica.wdw-labs.cloud" + multica_app_url: "https://multica.wdw-labs.cloud" + multica_version: "0.6.5" + multica_node_version: "24.22.1" + multica_workspaces_root: "/opt/multica/workspaces" + # No NAS mount under the workspaces root yet, so the mount guard is off. + # When utilidor is mounted, switch both lines together — never set the + # root onto a mount without also setting the guard, or a play that runs + # while the mount is down fills the root filesystem instead of failing. + # multica_workspaces_root: "/mnt/utilidor/multica/workspaces" + # multica_workspaces_root_mount: "/mnt/utilidor" + multica_agent_clis: + - claude + + # --- Secret source ---------------------------------------------------- + # This homelab keeps Ansible secrets in ansible-vault encrypted files, so + # that is the live form. vault_multica_pat is defined in + # group_vars/multica_agent_hosts/vault.yml (encrypted); the indirection + # keeps the name greppable and the file swappable. + multica_pat: "{{ vault_multica_pat }}" + # + # Alternatives, if the backend ever consolidates. Each is a one-line + # change here; the role is untouched by any of them. + # + # HashiCorp Vault (community.hashi_vault): + # multica_pat: "{{ lookup('community.hashi_vault.vault_kv2_get', + # 'multica/agent-host').secret.pat }}" + # + # 1Password (community.general): + # multica_pat: "{{ lookup('community.general.onepassword', + # 'Multica Agent Host PAT', field='credential') }}" + # + # Environment variable on the control node: + # multica_pat: "{{ lookup('env', 'MULTICA_PAT') }}" + + roles: + - role: multica_agent_host diff --git a/ansible/roles/multica_agent_host/.ansible-lint b/ansible/roles/multica_agent_host/.ansible-lint new file mode 100644 index 0000000..8439cd3 --- /dev/null +++ b/ansible/roles/multica_agent_host/.ansible-lint @@ -0,0 +1,9 @@ +--- +# The role's public variables are named in the issue's spec (multica_version, +# multica_server_url, multica_workspaces_root, multica_agent_clis, ...) and +# match the product's own namespace, including its MULTICA_* environment +# variables. Prefixing them with the role name instead would make every call +# site read multica_agent_host_multica_version. Internal facts use a leading +# underscore (_multica_*) to stay clearly private. +skip_list: + - var-naming[no-role-prefix] diff --git a/ansible/roles/multica_agent_host/README.md b/ansible/roles/multica_agent_host/README.md new file mode 100644 index 0000000..5900b1c --- /dev/null +++ b/ansible/roles/multica_agent_host/README.md @@ -0,0 +1,404 @@ +# multica_agent_host + +Provisions a Multica agent host from bare Linux to a **registered, running +daemon with its provider runtimes online**, and re-runs cleanly on a host that +is already provisioned. + +Reaches "daemon running, runtimes registered" unattended. The one step it +cannot do is the provider CLI's own subscription login, which needs a browser +once per host. + +## If your repo runs ansible-lint + +This role keeps the variable names from its spec (`multica_version`, +`multica_server_url`, `multica_workspaces_root`, ...), which match the +product's own `MULTICA_*` namespace. ansible-lint's `var-naming[no-role-prefix]` +wants them prefixed with the role name instead — `multica_agent_host_multica_version` +at every call site. + +The role ships the exemption at `roles/multica_agent_host/.ansible-lint`, but +**ansible-lint only reads that file from the directory you invoke it in**. If +you lint from your repo root, copy these two lines into your repo's own +`.ansible-lint`: + +```yaml +skip_list: + - var-naming[no-role-prefix] +``` + +Without it a repo-root lint run reports ~78 violations against a role that is +otherwise clean at the `production` profile. + +## What it does + +| Step | Handled | +|---|---| +| Dedicated `multica` service account (real home, linger) | yes | +| Node.js, pinned, from nodejs.org with SHASUMS256 verification | yes, when an npm-installed agent CLI is selected | +| `multica` CLI, pinned, from the GitHub release with `checksums.txt` verification | yes | +| Agent CLIs (`claude`, `hermes`, …) | yes | +| `workspaces_root`, on the intended mount, writable by the daemon user | yes | +| Multica authentication (`multica login --token`) | yes | +| CLI config (`server_url`, `app_url`, `workspaces_root`, `device_name`, …) | yes | +| Boot autostart (systemd user unit + `loginctl enable-linger`) | yes | +| Daemon start | yes | +| Runtime registration | follows automatically | +| **Provider CLI subscription login (e.g. `claude`)** | **no — browser, once per host** | +| Provider CLI credentials from a secret manager | out of scope; the role reports it as a remaining step | + +## Requirements + +- Linux target, `x86_64` or `aarch64`. +- systemd, for boot autostart. Without it the CLI falls back to an XDG autostart + entry, which does not help a headless host — set `multica_autostart: false` + and supervise the daemon yourself. +- Privilege escalation on the target (`become: true`). +- Ansible 2.14+. + +## Secrets + +**The role does not know where your secrets live.** It consumes one plain +variable: + +```yaml +multica_pat: "{{ ... }}" # mul_… user PAT, or mcn_… Cloud Node PAT +``` + +The playbook supplies it however your environment already does. In this +homelab that means an **`ansible-vault` encrypted var**, which is what the +example playbook uses; a `community.hashi_vault` lookup, a 1Password lookup and +a plain environment variable are written out beside it as commented +alternatives. + +See [Getting and storing the token](#getting-and-storing-the-token) below for +the end-to-end procedure. + +Three consequences worth stating: + +1. The role is reusable across environments that do not share a secrets backend. +2. Consolidating providers later is a one-line playbook change; this role is + untouched. +3. It sidesteps the HashiCorp Vault / `ansible-vault` ambiguity entirely — a + role that takes a value rather than fetching one works with either today. + +`no_log: true` is set on the login task regardless of where the token comes +from. Ansible logs command arguments by default; without it the PAT lands in +play output, in any CI log, and in every callback plugin. + +This matters just as much with `ansible-vault` as with a remote lookup: an +encrypted var is decrypted in memory for the duration of the play, so +encryption at rest does nothing for the runtime path. `no_log` is what covers +it. + +**Prefer a Cloud Node PAT (`mcn_…`) over a user PAT** where its scope is +narrower — a daemon host does not need everything a person can do. Confirm the +scope difference against your Multica deployment before deciding. + +## Getting and storing the token + +### Minting it + +**There is no CLI command for this.** `multica user` only manages your profile +and `multica auth` only reports status and logs out; nothing in the command +tree mints a token of either kind — the CLI only ever consumes one. A PAT is +created in the Multica web app under account settings, in a Tokens section. +(The exact menu path could not be confirmed from a headless runtime, so treat +that as "where to look", not a click path.) + +Two prefixes exist, and `multica login --token` accepts either: + +| Prefix | Kind | +|---|---| +| `mul_` | User PAT — acts as you, with everything your account can reach | +| `mcn_` | "Cloud Node" PAT | + +**On a self-hosted server, check whether `mcn_` is offered at all before +planning around it.** The string `mcn_` appears in exactly one place in the +shipped CLI — the `login --token` help text — and the only other place the +product uses "Cloud" as a noun is `multica setup cloud`, which points at +Multica Cloud (multica.ai) as opposed to `multica setup self-host`. That is +suggestive, not conclusive: Cloud Node PATs may well be a Multica Cloud +concept with no equivalent on a self-hosted deployment. The web UI's token +screen is the authority — whatever token types it offers you is the real +answer. + +Either prefix satisfies the role; preflight validates both. + +### Scoping a token on a self-hosted deployment + +If `mul_` is the only kind available, a user PAT on a long-lived daemon host +means every run that host claims carries your full authority, and revoking it +later cuts off your own access too. + +The lever that does exist is a **dedicated service member**. Workspace roles +are `owner`, `admin` and `member`: + +```bash +multica workspace member list --output json # see who exists and their roles +multica workspace member invite agents@example.com --role member +``` + +Have that account accept the invitation, log in as it, mint *its* PAT, and give +that to the host. `member` is coarser than a purpose-built node credential, but +it is strictly narrower than `owner`, and revoking it touches nothing you use. + +Do **not** reuse the session token your desktop CLI already holds. It is not a +PAT, the server renews it on its own schedule, and it ties the host's identity +to your desktop login — revoking one breaks the other. + +### Storing it with ansible-vault + +One-time, if you do not already have a vault password file: + +```bash +mkdir -p ~/.ansible +openssl rand -base64 48 > ~/.ansible/vault-pass-homelab +chmod 600 ~/.ansible/vault-pass-homelab +``` + +Point `ansible.cfg` at it so no play needs `--ask-vault-pass`: + +```ini +[defaults] +vault_password_file = ~/.ansible/vault-pass-homelab +``` + +Then create the encrypted var file: + +```bash +mkdir -p group_vars/multica_agent_hosts +ansible-vault create group_vars/multica_agent_hosts/vault.yml +``` + +with a single line in it: + +```yaml +vault_multica_pat: "mul_...." +``` + +The playbook already reads `multica_pat: "{{ vault_multica_pat }}"`. The +indirection is deliberate: the `vault_` prefix makes it obvious at a glance +which variables are encrypted, and it keeps the name greppable without +decrypting anything. + +**Alternative, if you would rather not keep a whole encrypted file:** encrypt +just the string and paste the block into an ordinary `group_vars` file. + +```bash +ansible-vault encrypt_string --stdin-name 'vault_multica_pat' +``` + +Paste the token, press Ctrl-D. Because it reads stdin, the token never enters +your shell history — which `ansible-vault encrypt_string 'mul_...'` as an +argument would. + +### Checks worth running once + +```bash +# the file is actually encrypted, not plaintext you forgot to encrypt +head -1 group_vars/multica_agent_hosts/vault.yml # => $ANSIBLE_VAULT;1.1;AES256 + +# the var resolves and the role's prefix check will pass, without printing it +ansible -i inventory/hosts.yml multica_agent_hosts -m debug \ + -a 'msg="pat resolves: {{ (vault_multica_pat | default("")) is match("^(mul_|mcn_)") }}"' +``` + +That second command prints only `true` or `false`, never the token. + +Add the vault password file to `.gitignore` if it lives anywhere near the repo. +The encrypted `vault.yml` is safe to commit; the password file never is. + +## Ansible owns the version + +`multica_version` is pinned and `disable_auto_update` is set to `true` in CLI +config. Re-running the play **is** the upgrade path. Without this the CLI +self-updates out from under the role's declared state and the host silently +stops matching what the repo says it is. + +The same reasoning applies to `multica_node_version` (full semver, not a major +track) and to `package_version` on each npm-installed agent CLI — left empty by +default, which tracks `latest`; set it to own that version too. + +## Variables + +See `defaults/main.yml` for the annotated list. The ones you will actually set: + +| Variable | Default | Notes | +|---|---|---| +| `multica_server_url` | — | **Required.** | +| `multica_pat` | — | **Required.** See above. | +| `multica_version` | `0.6.5` | Pinned CLI release. | +| `multica_node_version` | `24.22.1` | Pinned, full semver. | +| `multica_workspaces_root` | `/opt/multica/workspaces` | Local disk. Repoint at a mount when one exists. | +| `multica_workspaces_root_mount` | `""` | Set it and the play aborts if the mount is missing. | +| `multica_agent_clis` | `[claude]` | Keys of `multica_agent_cli_catalog`. | +| `multica_device_name` | short inventory hostname | Shows up in `runtime list` → `device_info`. | +| `multica_user` | `multica` | Daemon service account. | +| `multica_profile` | `""` | `--profile`, to isolate config/daemon state/workspaces. | +| `multica_allow_remote_install_scripts` | `false` | Required for `hermes`; see below. | + +### Adding an agent CLI + +Extend `multica_agent_cli_catalog`. Two install methods are implemented: + +```yaml +multica_agent_cli_catalog: + codex: + provider: codex # must match `provider` in `multica runtime list` + binary: codex + method: npm + package: "@openai/codex" + package_version: "" + version_command: "codex --version" + manual_login: true + manual_login_hint: "run `codex login` once as the daemon user" +``` + +`method: script` fetches `install_url` and runs it as the daemon user. That is a +supply-chain decision, so the role **refuses** unless +`multica_allow_remote_install_scripts: true`. `hermes` uses this method +(`https://hermes-agent.nousresearch.com/install.sh`). Vendoring the installer +into your repo and switching to a local `script` source is the stricter option. + +Agent CLI binaries are symlinked into `/usr/local/bin`. This matters: the daemon +runs from a systemd **user unit**, which gets systemd's default `PATH` and does +*not* source `/etc/profile.d`. A CLI reachable only from an interactive login +shell is invisible to the daemon. `MULTICA__PATH` is the escape hatch +if you need the binary somewhere else. + +Both install methods resolve the binary with `command -v` as the daemon user +and link *that* path, rather than assuming where they put it. This matters when +a CLI was already installed by other means — the install step is skipped, so +assuming the npm prefix would link to nothing. The role asserts the binary is +resolvable before linking, and linking the resolved path repairs a dangling +link left behind by an earlier run. + +## Moving the workspaces root onto a mount later + +`workspaces_root` holds per-run working directories, not durable state — the +daemon recreates what it needs. Moving it is therefore a drain-and-repoint, not +a migration: + +```bash +multica daemon disk-usage --output json # see what is actually there first +sudo -iu multica multica daemon stop # let in-flight runs finish first +``` + +Then set `multica_workspaces_root` and `multica_workspaces_root_mount` together +in the playbook and re-run. The role creates the new directory, proves the +daemon user can write to it, updates `config set workspaces_root`, and restarts +the daemon. + +**Set both variables in the same change.** Pointing the root at a mount path +without setting the guard means a play that runs while the mount is down +silently writes run workspaces onto the root filesystem — which nobody notices +until it fills. The guard turns that into a failed assertion instead. + +The old directory is left behind deliberately; remove it by hand once the new +one has taken runs. + +## Idempotency + +Every step reads before it writes: + +| Step | Guard | +|---|---| +| Node install | `node --version` equals the pin | +| CLI install | `multica --version` equals the pin | +| Agent CLI install | `command -v ` and its version command | +| Config | `multica config show` parsed and compared key by key | +| Login | `multica auth status` already names `multica_server_url` | +| Autostart | `multica daemon autostart status` | +| Daemon start | `multica daemon status --output json` reports `running` | + +A binary, config or credential change notifies a `multica daemon restart` +handler, which is flushed *before* verification reads the daemon's state back. + +A converged host reports zero changed tasks. + +## Verification + +The role ends in assertions, not hope: + +- `multica daemon status --output json` reports `status: running`, and its + `cli_version` and `server_url` match the declared values. +- `multica runtime list --output json` contains an **online** runtime for every + entry in `multica_agent_clis`, filtered to this host's `daemon_id`, with + `device_info` naming `multica_device_name`. Retried, because registration is + asynchronous after the daemon starts. +- `workspaces_root` exists, is a directory, is owned by the daemon user, is + actually writable by it (ownership is not writability on a root-squashed + network mount), and sits on `multica_workspaces_root_mount` when that is set. + +It finishes by printing the manual steps that remain — the per-host provider CLI +logins. + +## Testing against a throwaway VM + +Molecule is fine but a scratch VM is proportionate: + +```bash +# 1. clean host +ansible-playbook -i inventory/hosts.yml playbooks/provision-agent-host.yml -l scratch-vm + +# 2. re-run: must report changed=0 +ansible-playbook -i inventory/hosts.yml playbooks/provision-agent-host.yml -l scratch-vm + +# 3. reboot, then confirm the daemon came back on its own +ssh scratch-vm 'sudo -iu multica multica daemon status --output json' + +# 4. confirm from the workspace side +multica runtime list --output json | jq '.[] | select(.device_info | startswith("scratch-vm"))' +``` + +Then complete the browser login for each provider CLI once: + +```bash +ssh -t scratch-vm 'sudo -iu multica claude' +``` + +That step only becomes automatable if the host moves from a subscription to an +API key. + +## Known gaps + +- **Provider CLI login.** Unavoidable for subscription auth. +- **Hermes credentials.** Hermes reads 1Password natively; wiring that is not + implemented here because the right item layout is deployment-specific. The + role installs Hermes and reports the credential step as outstanding. +- **Non-systemd hosts.** Autostart degrades to an XDG entry that a headless host + never triggers. + +## Tags + +| Tag | Covers | +|---|---| +| `multica_workspaces` | workspaces root | +| `multica_node` | Node.js | +| `multica_cli` | multica CLI | +| `multica_agents` | agent CLIs | +| `multica_config` | `multica config set` | +| `multica_auth` | `multica login` | +| `multica_daemon` | autostart + start | +| `multica_verify` | the assertions | +| `multica` | all of the above | + +Preflight and the service-account step are tagged `always`: they build +`_multica_uid` and `_multica_user_env`, which every `become_user` task below +depends on, so a `--tags` subset still runs them. Both are idempotent. + +## Offline checks + +From the repo root: + +```bash +ansible-lint playbooks/provision-agent-host.yml roles/multica_agent_host +ansible-playbook --syntax-check playbooks/provision-agent-host.yml +ansible-playbook -i localhost, tests/expressions.yml +``` + +`tests/expressions.yml` runs the role's filter chains — `config show` parsing, +version extraction, the online-runtime difference, the mount containment check +— against fixtures captured from a real `multica` CLI, with no target host. +It catches the class of bug where a filter chain silently returns the wrong +type or matches nothing; it does not replace the scratch-VM run above. diff --git a/ansible/roles/multica_agent_host/defaults/main.yml b/ansible/roles/multica_agent_host/defaults/main.yml new file mode 100644 index 0000000..5eafec5 --- /dev/null +++ b/ansible/roles/multica_agent_host/defaults/main.yml @@ -0,0 +1,148 @@ +--- +# ============================================================================= +# multica_agent_host — defaults +# +# Everything the role needs is a plain variable. The role never reaches into a +# secrets backend itself; see `multica_pat` below. +# ============================================================================= + +# ----------------------------------------------------------------------------- +# Service account +# ----------------------------------------------------------------------------- +# The daemon runs as a dedicated unprivileged user with a real home directory. +# It needs a real home (CLI config, agent CLI credentials) and systemd linger, +# so this is NOT a `system: true` account. +multica_manage_user: true +multica_user: multica +multica_group: multica +multica_user_home: "/home/{{ multica_user }}" +multica_user_shell: /bin/bash +# Extra groups for the daemon user (e.g. docker). Left empty by default. +multica_user_extra_groups: [] + +# ----------------------------------------------------------------------------- +# multica CLI — Ansible owns the version +# ----------------------------------------------------------------------------- +# Pin it. Re-running the play IS the upgrade path; see multica_disable_auto_update. +multica_version: "0.6.5" +multica_install_dir: /usr/local/bin +multica_release_download_base_url: "https://github.com/multica-ai/multica/releases/download" +# Verify the release tarball against the release's own checksums.txt. +multica_verify_checksum: true + +# ----------------------------------------------------------------------------- +# Server / workspace +# ----------------------------------------------------------------------------- +multica_server_url: "" # REQUIRED, e.g. https://multica.wdw-labs.cloud +multica_app_url: "" # optional frontend URL +multica_workspace_id: "" # optional: pin this profile's default workspace +multica_profile: "" # optional: --profile (isolates config + daemon state) + +# ----------------------------------------------------------------------------- +# Daemon configuration (mirrors `multica config set` keys) +# ----------------------------------------------------------------------------- +# Local disk by default. Repoint this at a mount when one exists, and set +# multica_workspaces_root_mount at the same time so the play fails loudly if +# the mount is missing rather than silently filling the root filesystem. +multica_workspaces_root: "/opt/multica/workspaces" +# Optional guard: assert workspaces_root actually sits on this mount point. +# e.g. "/mnt/utilidor". Empty disables the check. +multica_workspaces_root_mount: "" +multica_workspaces_root_mode: "0750" + +multica_device_name: "{{ inventory_hostname.split('.')[0] }}" +multica_runtime_name: "" # "" = leave unset, CLI derives it +multica_max_concurrent_tasks: "" # "" = leave unset +multica_poll_interval: "" # "" = leave unset +multica_heartbeat_interval: "" # "" = leave unset +multica_agent_timeout: "" # "" = leave unset ("0s" disables the cap) + +# Ansible owns the version: stop the CLI self-updating out from under the +# role's declared state. Set false only if you want the daemon to self-update. +multica_disable_auto_update: true + +# Boot autostart. On Linux `multica daemon autostart enable` writes a systemd +# *user* unit, which only starts at boot when the user has lingering enabled. +multica_autostart: true +multica_enable_linger: true + +# ----------------------------------------------------------------------------- +# Authentication — backend-agnostic by design +# ----------------------------------------------------------------------------- +# REQUIRED. A Multica personal access token: `mul_...` (user PAT) or +# `mcn_...` (Cloud Node PAT). The role takes a VALUE; it does not know or care +# where it came from. Supply it from the playbook with whatever your +# environment already uses: +# +# HashiCorp Vault: +# multica_pat: "{{ lookup('community.hashi_vault.vault_kv2_get', +# 'multica/agent-host').secret.pat }}" +# ansible-vault encrypted var: +# multica_pat: "{{ vaulted_multica_pat }}" +# 1Password: +# multica_pat: "{{ lookup('community.general.onepassword', +# 'Multica Agent Host PAT', field='credential') }}" +# Environment: +# multica_pat: "{{ lookup('env', 'MULTICA_PAT') }}" +# +# Consolidating secret providers later is a one-line playbook change; this role +# is untouched by it. +multica_pat: "" + +# Force `multica login --token` even when the host already reports an +# authenticated session against multica_server_url. +multica_force_relogin: false + +# ----------------------------------------------------------------------------- +# Agent CLIs +# ----------------------------------------------------------------------------- +# Which provider CLIs this host should offer. Each must be a key in +# multica_agent_cli_catalog. The daemon registers one runtime per CLI it finds. +multica_agent_clis: + - claude + +# Some agent CLIs are installed by piping a vendor script into a shell. That is +# a real supply-chain decision, so it is opt-in and off by default. +multica_allow_remote_install_scripts: false + +multica_agent_cli_catalog: + claude: + provider: claude # matches `provider` in `multica runtime list` + binary: claude + method: npm + package: "@anthropic-ai/claude-code" + # Pinned: npm "latest" is 2.1.306 while "stable" is 2.1.304, so an empty + # value silently tracks whichever npm serves that day. 2.1.304 is the + # version running on haunted-mansion. + package_version: "2.1.304" + version_command: "claude --version" + # Subscription auth needs a browser once per host. Not automatable here. + manual_login: true + manual_login_hint: "run `claude` once as {{ multica_user }} and complete the login" + hermes: + provider: hermes + binary: hermes + method: script + install_url: "https://hermes-agent.nousresearch.com/install.sh" + install_dir: "{{ multica_user_home }}/.hermes/hermes-agent" + version_command: "hermes --version" + manual_login: true + manual_login_hint: "configure Hermes credentials as {{ multica_user }} (Hermes reads 1Password natively; confirm against its docs)" + +# ----------------------------------------------------------------------------- +# Node.js (installed only when a selected agent CLI needs npm) +# ----------------------------------------------------------------------------- +# Full semver — this is a pin, not a major-version track. +multica_node_version: "24.22.1" +multica_node_dist_base_url: "https://nodejs.org/dist" +multica_node_prefix: /usr/local/lib/nodejs +# npm global prefix for the daemon user, so agent CLI installs need no root. +multica_npm_prefix: "{{ multica_user_home }}/.npm-global" + +# ----------------------------------------------------------------------------- +# Verification +# ----------------------------------------------------------------------------- +multica_verify: true +# How long to wait for runtimes to appear online after the daemon starts. +multica_verify_retries: 24 +multica_verify_delay: 5 diff --git a/ansible/roles/multica_agent_host/handlers/main.yml b/ansible/roles/multica_agent_host/handlers/main.yml new file mode 100644 index 0000000..cdd73d2 --- /dev/null +++ b/ansible/roles/multica_agent_host/handlers/main.yml @@ -0,0 +1,9 @@ +--- +- name: Restart multica daemon + ansible.builtin.command: + argv: "{{ _multica_cmd + ['daemon', 'restart'] }}" + become: true + become_user: "{{ multica_user }}" + environment: "{{ _multica_user_env }}" + changed_when: true + listen: restart multica daemon diff --git a/ansible/roles/multica_agent_host/meta/main.yml b/ansible/roles/multica_agent_host/meta/main.yml new file mode 100644 index 0000000..0fba09a --- /dev/null +++ b/ansible/roles/multica_agent_host/meta/main.yml @@ -0,0 +1,22 @@ +--- +galaxy_info: + role_name: multica_agent_host + namespace: wdwlabs + author: WDW Labs + description: >- + Provision a Multica agent host from bare Linux to a registered, running + daemon with its provider runtimes online. + license: MIT + min_ansible_version: "2.14" + platforms: + - name: Debian + versions: [bullseye, bookworm, trixie] + - name: Ubuntu + versions: [jammy, noble] + - name: EL + versions: ["9"] + galaxy_tags: + - multica + - agents + - daemon +dependencies: [] diff --git a/ansible/roles/multica_agent_host/tasks/agent_cli/_link.yml b/ansible/roles/multica_agent_host/tasks/agent_cli/_link.yml new file mode 100644 index 0000000..b387b65 --- /dev/null +++ b/ansible/roles/multica_agent_host/tasks/agent_cli/_link.yml @@ -0,0 +1,42 @@ +--- +# Resolve where the binary ACTUALLY is before linking to it. +# +# The npm path used to link unconditionally from multica_npm_prefix. When the +# CLI was already installed by some other means — a distro package, a vendor +# installer, a previous hand-rollout — nothing was installed at that prefix and +# `file: state=link force=true` happily created a DANGLING symlink in +# /usr/local/bin. PATH resolution skips a dangling link, so discovery still +# worked by luck wherever the real binary was already on systemd's PATH, and +# the role's actual guarantee silently did not hold. +- name: "Agents | Resolve the installed binary | {{ _agent_key }}" + ansible.builtin.command: + argv: ["/bin/sh", "-c", "command -v {{ _agent.binary }}"] + become: true + become_user: "{{ multica_user }}" + environment: "{{ _multica_user_env }}" + register: _agent_which + changed_when: false + failed_when: false + check_mode: false + +- name: "Agents | Require the binary to be resolvable | {{ _agent_key }}" + ansible.builtin.assert: + that: + - _agent_which.rc == 0 + - (_agent_which.stdout | trim) | length > 0 + fail_msg: >- + {{ _agent.binary }} is not on {{ multica_user }}'s PATH after the install + step. The daemon discovers provider CLIs by PATH, so it would register no + {{ _agent_key }} runtime. + quiet: true + +# Linking the resolved path also repairs a dangling link left by an older run: +# `command -v` skips the broken link, so this overwrites it with a working one. +- name: "Agents | Link onto the default PATH | {{ _agent_key }}" + ansible.builtin.file: + src: "{{ _agent_which.stdout | trim }}" + dest: "{{ multica_install_dir }}/{{ _agent.binary }}" + state: link + force: true + become: true + when: (_agent_which.stdout | trim) != (multica_install_dir ~ '/' ~ _agent.binary) diff --git a/ansible/roles/multica_agent_host/tasks/agent_cli/npm.yml b/ansible/roles/multica_agent_host/tasks/agent_cli/npm.yml new file mode 100644 index 0000000..7cf4389 --- /dev/null +++ b/ansible/roles/multica_agent_host/tasks/agent_cli/npm.yml @@ -0,0 +1,46 @@ +--- +- name: "Agents | Resolve catalog entry | {{ _agent_key }}" + ansible.builtin.set_fact: + _agent: "{{ multica_agent_cli_catalog[_agent_key] }}" + +- name: "Agents | Read the installed version | {{ _agent_key }}" + ansible.builtin.command: + argv: ["/bin/sh", "-c", "command -v {{ _agent.binary }} >/dev/null 2>&1 && {{ _agent.version_command }}"] + become: true + become_user: "{{ multica_user }}" + environment: "{{ _multica_user_env }}" + register: _agent_installed + changed_when: false + failed_when: false + check_mode: false + +# Version formats differ per CLI ("2.1.304 (Claude Code)", "Hermes Agent +# v0.21.5+..."), so this matches the pinned version anywhere in the output — +# but on a version-component boundary. A plain substring test would accept +# 2.1.3045 for a pin of 2.1.304, and silently never upgrade. +- name: "Agents | Decide whether an install is needed | {{ _agent_key }}" + ansible.builtin.set_fact: + _agent_needs_install: >- + {{ _agent_installed.rc != 0 + or (_agent.package_version | default('') | length > 0 + and not ((_agent_installed.stdout | default('')) is search( + '(^|[^0-9.])' ~ (_agent.package_version | regex_escape) ~ '([^0-9.]|$)'))) }} + +- name: "Agents | Install via npm as the daemon user | {{ _agent_key }}" + ansible.builtin.command: + argv: + - "{{ multica_install_dir }}/npm" + - install + - --global + - --no-fund + - --no-audit + - "{{ _agent.package }}{{ ('@' ~ _agent.package_version) if _agent.package_version | default('') | length > 0 else '' }}" + become: true + become_user: "{{ multica_user }}" + environment: "{{ _multica_user_env }}" + when: _agent_needs_install | bool + changed_when: true + notify: restart multica daemon + +- name: "Agents | Resolve and link onto the default PATH | {{ _agent_key }}" + ansible.builtin.include_tasks: _link.yml diff --git a/ansible/roles/multica_agent_host/tasks/agent_cli/script.yml b/ansible/roles/multica_agent_host/tasks/agent_cli/script.yml new file mode 100644 index 0000000..1b56bd9 --- /dev/null +++ b/ansible/roles/multica_agent_host/tasks/agent_cli/script.yml @@ -0,0 +1,53 @@ +--- +- name: "Agents | Resolve catalog entry | {{ _agent_key }}" + ansible.builtin.set_fact: + _agent: "{{ multica_agent_cli_catalog[_agent_key] }}" + +- name: "Agents | Read the installed version | {{ _agent_key }}" + ansible.builtin.command: + argv: ["/bin/sh", "-c", "command -v {{ _agent.binary }} >/dev/null 2>&1 && {{ _agent.version_command }}"] + become: true + become_user: "{{ multica_user }}" + environment: "{{ _multica_user_env }}" + register: _agent_installed + changed_when: false + failed_when: false + check_mode: false + +# Piping a vendor script into a shell is a supply-chain decision, so the role +# refuses rather than making it quietly on the operator's behalf. +- name: "Agents | Refuse to run a remote installer unless allowed | {{ _agent_key }}" + ansible.builtin.fail: + msg: >- + {{ _agent_key }} installs by executing {{ _agent.install_url }}. + Set multica_allow_remote_install_scripts: true to permit that, vendor the + installer yourself, or drop {{ _agent_key }} from multica_agent_clis. + when: + - _agent_installed.rc != 0 + - not (multica_allow_remote_install_scripts | bool) + +- name: "Agents | Install via the vendor script | {{ _agent_key }}" + when: + - _agent_installed.rc != 0 + - multica_allow_remote_install_scripts | bool + block: + - name: "Agents | Fetch the installer | {{ _agent_key }}" + ansible.builtin.get_url: + url: "{{ _agent.install_url }}" + dest: "{{ multica_user_home }}/.cache/{{ _agent_key }}-install.sh" + owner: "{{ multica_user }}" + group: "{{ multica_group }}" + mode: "0700" + become: true + + - name: "Agents | Run it as the daemon user | {{ _agent_key }}" + ansible.builtin.command: + argv: ["/bin/bash", "{{ multica_user_home }}/.cache/{{ _agent_key }}-install.sh"] + become: true + become_user: "{{ multica_user }}" + environment: "{{ _multica_user_env }}" + changed_when: true + notify: restart multica daemon + +- name: "Agents | Resolve and link onto the default PATH | {{ _agent_key }}" + ansible.builtin.include_tasks: _link.yml diff --git a/ansible/roles/multica_agent_host/tasks/agent_clis.yml b/ansible/roles/multica_agent_host/tasks/agent_clis.yml new file mode 100644 index 0000000..7647732 --- /dev/null +++ b/ansible/roles/multica_agent_host/tasks/agent_clis.yml @@ -0,0 +1,7 @@ +--- +- name: Agents | Install each selected agent CLI + ansible.builtin.include_tasks: "agent_cli/{{ multica_agent_cli_catalog[_agent_key].method }}.yml" + loop: "{{ multica_agent_clis }}" + loop_control: + loop_var: _agent_key + label: "{{ _agent_key }}" diff --git a/ansible/roles/multica_agent_host/tasks/auth.yml b/ansible/roles/multica_agent_host/tasks/auth.yml new file mode 100644 index 0000000..d5fb803 --- /dev/null +++ b/ansible/roles/multica_agent_host/tasks/auth.yml @@ -0,0 +1,44 @@ +--- +# `multica auth status` prints the server and user when authenticated and exits +# non-zero otherwise, so it is the idempotency guard for login. +- name: Auth | Read the current authentication state + ansible.builtin.command: + argv: "{{ _multica_cmd + ['auth', 'status'] }}" + become: true + become_user: "{{ multica_user }}" + environment: "{{ _multica_user_env }}" + register: _multica_auth_status + changed_when: false + failed_when: false + check_mode: false + +- name: Auth | Decide whether a login is needed + ansible.builtin.set_fact: + _multica_needs_login: >- + {{ multica_force_relogin | bool + or _multica_auth_status.rc != 0 + or multica_server_url.rstrip('/') not in (_multica_auth_status.stdout | default('')) }} + +# no_log is mandatory: without it Ansible records the full argv — including the +# PAT — in play output, CI logs, and every callback plugin. +- name: Auth | Log in with the personal access token + ansible.builtin.command: + argv: "{{ _multica_cmd + ['login', '--token', multica_pat] }}" + become: true + become_user: "{{ multica_user }}" + environment: "{{ _multica_user_env }}" + when: _multica_needs_login | bool + changed_when: true + no_log: true + notify: restart multica daemon + +- name: Auth | Pin the default workspace + ansible.builtin.command: + argv: "{{ _multica_cmd + ['workspace', 'switch', multica_workspace_id] }}" + become: true + become_user: "{{ multica_user }}" + environment: "{{ _multica_user_env }}" + when: multica_workspace_id | length > 0 + register: _multica_ws_switch + changed_when: _multica_needs_login | bool + notify: restart multica daemon diff --git a/ansible/roles/multica_agent_host/tasks/config.yml b/ansible/roles/multica_agent_host/tasks/config.yml new file mode 100644 index 0000000..519f866 --- /dev/null +++ b/ansible/roles/multica_agent_host/tasks/config.yml @@ -0,0 +1,56 @@ +--- +# The CLI owns its config file format, so the role drives `multica config set` +# rather than templating config.json. Each key is read back first, so a +# converged host reports zero changes. + +- name: Config | Build the desired configuration + ansible.builtin.set_fact: + _multica_desired_config: >- + {{ { + 'server_url': multica_server_url, + 'app_url': multica_app_url, + 'workspaces_root': multica_workspaces_root, + 'device_name': multica_device_name, + 'runtime_name': multica_runtime_name, + 'max_concurrent_tasks': multica_max_concurrent_tasks, + 'poll_interval': multica_poll_interval, + 'heartbeat_interval': multica_heartbeat_interval, + 'agent_timeout': multica_agent_timeout, + 'disable_auto_update': ('true' if multica_disable_auto_update | bool else 'false'), + } | dict2items + | rejectattr('value', 'equalto', '') + | list | items2dict }} + +- name: Config | Read the current configuration + ansible.builtin.command: + argv: "{{ _multica_cmd + ['config', 'show'] }}" + become: true + become_user: "{{ multica_user }}" + environment: "{{ _multica_user_env }}" + register: _multica_config_show + changed_when: false + check_mode: false + +# `config show` prints "key:value", with "(not set)" for unset keys. +- name: Config | Parse it + ansible.builtin.set_fact: + _multica_current_config: >- + {{ dict( + _multica_config_show.stdout_lines + | select('match', '^[a-z_]+:') + | map('regex_replace', '^([a-z_]+):\s*(.*?)\s*$', '\1|\2') + | map('split', '|') + ) }} + +- name: Config | Apply the keys that differ + ansible.builtin.command: + argv: "{{ _multica_cmd + ['config', 'set', item.key, item.value] }}" + become: true + become_user: "{{ multica_user }}" + environment: "{{ _multica_user_env }}" + loop: "{{ _multica_desired_config | dict2items }}" + loop_control: + label: "{{ item.key }}={{ item.value }}" + when: (_multica_current_config[item.key] | default('(not set)')) != item.value + changed_when: true + notify: restart multica daemon diff --git a/ansible/roles/multica_agent_host/tasks/daemon.yml b/ansible/roles/multica_agent_host/tasks/daemon.yml new file mode 100644 index 0000000..bbd4f31 --- /dev/null +++ b/ansible/roles/multica_agent_host/tasks/daemon.yml @@ -0,0 +1,73 @@ +--- +# On Linux `multica daemon autostart enable` writes a systemd *user* unit. +# A user unit does not start at boot unless the user lingers — without this the +# daemon only comes back when someone logs in, which on a headless agent host +# is never. +- name: Daemon | Enable systemd lingering for the daemon user + ansible.builtin.command: + argv: ["loginctl", "enable-linger", "{{ multica_user }}"] + creates: "/var/lib/systemd/linger/{{ multica_user }}" + become: true + when: + - multica_enable_linger | bool + - multica_autostart | bool + +- name: Daemon | Wait for the user's runtime directory + ansible.builtin.wait_for: + path: "/run/user/{{ _multica_uid }}" + state: present + timeout: 30 + become: true + when: + - multica_enable_linger | bool + - multica_autostart | bool + +- name: Daemon | Read the current autostart registration + ansible.builtin.command: + argv: "{{ _multica_cmd + ['daemon', 'autostart', 'status'] }}" + become: true + become_user: "{{ multica_user }}" + environment: "{{ _multica_user_env }}" + register: _multica_autostart_status + changed_when: false + failed_when: false + check_mode: false + +- name: Daemon | Register boot autostart + ansible.builtin.command: + argv: "{{ _multica_cmd + ['daemon', 'autostart', 'enable'] }}" + become: true + become_user: "{{ multica_user }}" + environment: "{{ _multica_user_env }}" + when: + - multica_autostart | bool + - _multica_autostart_status.rc != 0 + or 'enabled' not in (_multica_autostart_status.stdout | default('') | lower) + changed_when: true + +- name: Daemon | Read daemon status + ansible.builtin.command: + argv: "{{ _multica_cmd + ['daemon', 'status', '--output', 'json'] }}" + become: true + become_user: "{{ multica_user }}" + environment: "{{ _multica_user_env }}" + register: _multica_daemon_status + changed_when: false + failed_when: false + check_mode: false + +- name: Daemon | Start it if it is not already running + ansible.builtin.command: + argv: "{{ _multica_cmd + ['daemon', 'start'] }}" + become: true + become_user: "{{ multica_user }}" + environment: "{{ _multica_user_env }}" + when: >- + _multica_daemon_status.rc != 0 + or ((_multica_daemon_status.stdout | from_json).status | default('')) != 'running' + changed_when: true + +# Anything that changed the binary, the config or the credentials restarts the +# daemon here, before verification reads its state back. +- name: Daemon | Flush handlers + ansible.builtin.meta: flush_handlers diff --git a/ansible/roles/multica_agent_host/tasks/main.yml b/ansible/roles/multica_agent_host/tasks/main.yml new file mode 100644 index 0000000..55e4db6 --- /dev/null +++ b/ansible/roles/multica_agent_host/tasks/main.yml @@ -0,0 +1,45 @@ +--- +- name: Preflight + ansible.builtin.import_tasks: preflight.yml + # Prerequisite for every other block: always runs, even under a --tags subset. + tags: [always] + +- name: Service account + ansible.builtin.import_tasks: user.yml + # Builds _multica_uid / _multica_user_env, which every later become_user task + # needs; creating an existing user is a no-op. + tags: [always] + +- name: Workspaces root + ansible.builtin.import_tasks: workspaces_root.yml + tags: [multica, multica_workspaces] + +- name: Node.js + ansible.builtin.import_tasks: node.yml + when: _multica_node_required | bool + tags: [multica, multica_node] + +- name: Multica CLI + ansible.builtin.import_tasks: multica_cli.yml + tags: [multica, multica_cli] + +- name: Agent CLIs + ansible.builtin.import_tasks: agent_clis.yml + tags: [multica, multica_agents] + +- name: CLI configuration + ansible.builtin.import_tasks: config.yml + tags: [multica, multica_config] + +- name: Authentication + ansible.builtin.import_tasks: auth.yml + tags: [multica, multica_auth] + +- name: Daemon + ansible.builtin.import_tasks: daemon.yml + tags: [multica, multica_daemon] + +- name: Verify + ansible.builtin.import_tasks: verify.yml + when: multica_verify | bool + tags: [multica, multica_verify] diff --git a/ansible/roles/multica_agent_host/tasks/multica_cli.yml b/ansible/roles/multica_agent_host/tasks/multica_cli.yml new file mode 100644 index 0000000..316bdff --- /dev/null +++ b/ansible/roles/multica_agent_host/tasks/multica_cli.yml @@ -0,0 +1,72 @@ +--- +- name: CLI | Check whether the binary is already present + ansible.builtin.stat: + path: "{{ _multica_bin }}" + register: _multica_cli_stat + check_mode: false + +- name: CLI | Read the installed version + ansible.builtin.command: + argv: ["{{ _multica_bin }}", "--version"] + register: _multica_cli_installed + when: _multica_cli_stat.stat.exists + changed_when: false + failed_when: false + check_mode: false + +# `multica --version` prints e.g. "multica 0.6.5 (commit: ..., built: ...)". +- name: CLI | Extract the installed version string + ansible.builtin.set_fact: + _multica_cli_version_found: >- + {{ ((_multica_cli_installed.stdout_lines | default([]) | first | default('')).split() + ['', ''])[1] + if (_multica_cli_installed.rc | default(1)) == 0 else '' }} + +- name: CLI | Decide whether an install is needed + ansible.builtin.set_fact: + _multica_cli_needs_install: "{{ (_multica_cli_version_found | trim) != multica_version }}" + +- name: CLI | Install the pinned release + when: _multica_cli_needs_install | bool + block: + - name: CLI | Create a staging directory + ansible.builtin.tempfile: + state: directory + suffix: multica-cli + register: _multica_stage + become: true + + - name: CLI | Download the release tarball + ansible.builtin.get_url: + url: "{{ _multica_release_url }}" + dest: "{{ _multica_stage.path }}/{{ _multica_tarball }}" + checksum: "{{ ('sha256:' ~ _multica_checksums_url) if multica_verify_checksum | bool else omit }}" + mode: "0644" + become: true + + - name: CLI | Unpack the binary + ansible.builtin.unarchive: + src: "{{ _multica_stage.path }}/{{ _multica_tarball }}" + dest: "{{ _multica_stage.path }}" + remote_src: true + become: true + + # Replacing the binary while the daemon runs is fine — the handler restarts + # it, and the daemon also watches its own binary for version changes. + - name: CLI | Install it onto the PATH + ansible.builtin.copy: + src: "{{ _multica_stage.path }}/multica" + dest: "{{ _multica_bin }}" + remote_src: true + owner: root + group: root + mode: "0755" + become: true + notify: restart multica daemon + + always: + - name: CLI | Remove the staging directory + ansible.builtin.file: + path: "{{ _multica_stage.path }}" + state: absent + become: true + when: _multica_stage.path is defined diff --git a/ansible/roles/multica_agent_host/tasks/node.yml b/ansible/roles/multica_agent_host/tasks/node.yml new file mode 100644 index 0000000..caded30 --- /dev/null +++ b/ansible/roles/multica_agent_host/tasks/node.yml @@ -0,0 +1,93 @@ +--- +# Installed from the official nodejs.org tarball rather than a distro repo so +# that multica_node_version is a real pin on every distribution. + +- name: Node | Check whether the pinned runtime is already present + ansible.builtin.stat: + path: "{{ multica_install_dir }}/node" + register: _multica_node_stat + check_mode: false + +- name: Node | Read the installed version + ansible.builtin.command: + argv: ["{{ multica_install_dir }}/node", "--version"] + register: _multica_node_installed + when: _multica_node_stat.stat.exists + changed_when: false + failed_when: false + check_mode: false + +- name: Node | Decide whether an install is needed + ansible.builtin.set_fact: + _multica_node_needs_install: >- + {{ (not _multica_node_stat.stat.exists) + or (_multica_node_installed.rc | default(1)) != 0 + or (_multica_node_installed.stdout | default('') | trim) != ('v' ~ multica_node_version) }} + +- name: Node | Install the pinned runtime + when: _multica_node_needs_install | bool + block: + - name: Node | Create the install prefix + ansible.builtin.file: + path: "{{ multica_node_prefix }}" + state: directory + owner: root + group: root + mode: "0755" + become: true + + # get_url resolves `sha256:` against the file's basename in SHASUMS256.txt. + - name: Node | Download the tarball with checksum verification + ansible.builtin.get_url: + url: "{{ multica_node_dist_base_url }}/v{{ multica_node_version }}/node-v{{ multica_node_version }}-linux-{{ _multica_node_arch }}.tar.xz" + dest: "/tmp/node-v{{ multica_node_version }}-linux-{{ _multica_node_arch }}.tar.xz" + checksum: "sha256:{{ multica_node_dist_base_url }}/v{{ multica_node_version }}/SHASUMS256.txt" + mode: "0644" + become: true + + - name: Node | Unpack it + ansible.builtin.unarchive: + src: "/tmp/node-v{{ multica_node_version }}-linux-{{ _multica_node_arch }}.tar.xz" + dest: "{{ multica_node_prefix }}" + remote_src: true + owner: root + group: root + creates: "{{ multica_node_prefix }}/node-v{{ multica_node_version }}-linux-{{ _multica_node_arch }}/bin/node" + become: true + + - name: Node | Remove the downloaded tarball + ansible.builtin.file: + path: "/tmp/node-v{{ multica_node_version }}-linux-{{ _multica_node_arch }}.tar.xz" + state: absent + become: true + +# Symlinked into /usr/local/bin, which is on systemd's default PATH — the +# daemon's autostart unit does not source /etc/profile.d. +- name: Node | Link node, npm and npx onto the default PATH + ansible.builtin.file: + src: "{{ multica_node_prefix }}/node-v{{ multica_node_version }}-linux-{{ _multica_node_arch }}/bin/{{ item }}" + dest: "{{ multica_install_dir }}/{{ item }}" + state: link + force: true + become: true + loop: [node, npm, npx] + +- name: Node | Create the daemon user's npm global prefix + ansible.builtin.file: + path: "{{ multica_npm_prefix }}" + state: directory + owner: "{{ multica_user }}" + group: "{{ multica_group }}" + mode: "0755" + become: true + +- name: Node | Point the daemon user's npm at that prefix + ansible.builtin.lineinfile: + path: "{{ multica_user_home }}/.npmrc" + regexp: '^prefix=' + line: "prefix={{ multica_npm_prefix }}" + create: true + owner: "{{ multica_user }}" + group: "{{ multica_group }}" + mode: "0644" + become: true diff --git a/ansible/roles/multica_agent_host/tasks/preflight.yml b/ansible/roles/multica_agent_host/tasks/preflight.yml new file mode 100644 index 0000000..eac3f11 --- /dev/null +++ b/ansible/roles/multica_agent_host/tasks/preflight.yml @@ -0,0 +1,86 @@ +--- +- name: Preflight | Require a Linux target + ansible.builtin.assert: + that: + - ansible_facts['system'] == 'Linux' + fail_msg: >- + multica_agent_host supports Linux hosts only + (found {{ ansible_facts['system'] }}). + quiet: true + +- name: Preflight | Require a supported CPU architecture + ansible.builtin.assert: + that: + - ansible_facts['architecture'] in _multica_arch_map + fail_msg: >- + No multica CLI release asset for architecture + {{ ansible_facts['architecture'] }}. + Supported: {{ _multica_arch_map.keys() | list | join(', ') }}. + quiet: true + +- name: Preflight | Require the server URL + ansible.builtin.assert: + that: + - multica_server_url | length > 0 + - multica_server_url is match('^https?://') + fail_msg: "multica_server_url must be set to an http(s) URL." + quiet: true + +# no_log: a failed assert would otherwise render the variables it inspected. +- name: Preflight | Require a Multica personal access token + ansible.builtin.assert: + that: + - multica_pat | length > 0 + - multica_pat is match('^(mul_|mcn_)') + fail_msg: >- + multica_pat must be set to a Multica personal access token + (mul_... user PAT, or mcn_... Cloud Node PAT). Supply it from the + playbook; see this role's README for Vault / 1Password / env examples. + quiet: true + no_log: true + +- name: Preflight | Require every selected agent CLI to be in the catalog + ansible.builtin.assert: + that: + - multica_agent_clis | difference(multica_agent_cli_catalog.keys() | list) | length == 0 + fail_msg: >- + Unknown agent CLI(s): + {{ multica_agent_clis | difference(multica_agent_cli_catalog.keys() | list) | join(', ') }}. + Add them to multica_agent_cli_catalog or drop them from multica_agent_clis. + quiet: true + +- name: Preflight | Require at least one agent CLI + ansible.builtin.assert: + that: + - multica_agent_clis | length > 0 + fail_msg: >- + multica_agent_clis is empty; the daemon would register no runtimes and + the host could never claim a run. + quiet: true + +- name: Preflight | Resolve release artefact names and the invocation prefix + ansible.builtin.set_fact: + _multica_arch: "{{ _multica_arch_map[ansible_facts['architecture']] }}" + _multica_node_arch: "{{ _multica_node_arch_map[ansible_facts['architecture']] }}" + _multica_bin: "{{ multica_install_dir }}/multica" + _multica_cmd: >- + {{ [multica_install_dir ~ '/multica'] + + (['--profile', multica_profile] if multica_profile | length > 0 else []) }} + _multica_node_required: >- + {{ multica_agent_clis + | map('extract', multica_agent_cli_catalog, 'method') + | select('equalto', 'npm') | list | length > 0 }} + +- name: Preflight | Name the release tarball + ansible.builtin.set_fact: + _multica_tarball: "multica-cli-{{ multica_version }}-linux-{{ _multica_arch }}.tar.gz" + _multica_release_url: >- + {{ multica_release_download_base_url }}/v{{ multica_version }}/multica-cli-{{ multica_version }}-linux-{{ _multica_arch }}.tar.gz + _multica_checksums_url: >- + {{ multica_release_download_base_url }}/v{{ multica_version }}/checksums.txt + +- name: Preflight | Require base tooling on the target + ansible.builtin.package: + name: "{{ ['tar', 'xz-utils' if ansible_facts['os_family'] == 'Debian' else 'xz', 'ca-certificates', 'git'] }}" + state: present + become: true diff --git a/ansible/roles/multica_agent_host/tasks/user.yml b/ansible/roles/multica_agent_host/tasks/user.yml new file mode 100644 index 0000000..c95be5a --- /dev/null +++ b/ansible/roles/multica_agent_host/tasks/user.yml @@ -0,0 +1,63 @@ +--- +- name: User | Create the daemon group + ansible.builtin.group: + name: "{{ multica_group }}" + state: present + become: true + when: multica_manage_user | bool + +# Deliberately not a system account: the daemon needs a real home for CLI +# config and agent CLI credentials, and systemd linger needs a real user. +- name: User | Create the daemon user + ansible.builtin.user: + name: "{{ multica_user }}" + group: "{{ multica_group }}" + groups: "{{ multica_user_extra_groups }}" + append: true + home: "{{ multica_user_home }}" + shell: "{{ multica_user_shell }}" + create_home: true + state: present + become: true + when: multica_manage_user | bool + +- name: User | Look up the daemon user + ansible.builtin.getent: + database: passwd + key: "{{ multica_user }}" + become: true + +- name: User | Record uid and the per-user runtime environment + ansible.builtin.set_fact: + _multica_uid: "{{ ansible_facts.getent_passwd[multica_user][1] }}" + +# systemd user units — which is what `multica daemon autostart enable` writes on +# Linux — are unreachable over a plain `sudo -u` shell without these. Without +# XDG_RUNTIME_DIR, `systemctl --user` fails with "Failed to connect to bus". +- name: User | Build the daemon user's command environment + ansible.builtin.set_fact: + _multica_user_env: + HOME: "{{ multica_user_home }}" + USER: "{{ multica_user }}" + LOGNAME: "{{ multica_user }}" + XDG_RUNTIME_DIR: "/run/user/{{ _multica_uid }}" + DBUS_SESSION_BUS_ADDRESS: "unix:path=/run/user/{{ _multica_uid }}/bus" + PATH: "{{ multica_npm_prefix }}/bin:{{ multica_user_home }}/.local/bin:{{ multica_install_dir }}:/usr/local/bin:/usr/bin:/bin" + +- name: User | Ensure the user's local bin directory exists + ansible.builtin.file: + path: "{{ multica_user_home }}/.local/bin" + state: directory + owner: "{{ multica_user }}" + group: "{{ multica_group }}" + mode: "0755" + become: true + +- name: User | Ensure the user's cache directory exists + ansible.builtin.file: + path: "{{ multica_user_home }}/.cache" + state: directory + owner: "{{ multica_user }}" + group: "{{ multica_group }}" + mode: "0700" + become: true diff --git a/ansible/roles/multica_agent_host/tasks/verify.yml b/ansible/roles/multica_agent_host/tasks/verify.yml new file mode 100644 index 0000000..b91dc25 --- /dev/null +++ b/ansible/roles/multica_agent_host/tasks/verify.yml @@ -0,0 +1,120 @@ +--- +# Assertions, not hope. Each of these has failed in practice for a different +# reason, so each gets its own check and its own error message. + +- name: Verify | Daemon reports running + ansible.builtin.command: + argv: "{{ _multica_cmd + ['daemon', 'status', '--output', 'json'] }}" + become: true + become_user: "{{ multica_user }}" + environment: "{{ _multica_user_env }}" + register: _multica_verify_daemon + until: + - _multica_verify_daemon.rc == 0 + - (_multica_verify_daemon.stdout | from_json).status | default('') == 'running' + retries: "{{ multica_verify_retries }}" + delay: "{{ multica_verify_delay }}" + changed_when: false + check_mode: false + +- name: Verify | Daemon is the pinned version and sees the configured workspaces root + ansible.builtin.assert: + that: + - (_multica_verify_daemon.stdout | from_json).cli_version == multica_version + - (_multica_verify_daemon.stdout | from_json).server_url | regex_replace('/$', '') + == multica_server_url | regex_replace('/$', '') + fail_msg: >- + Daemon is running but does not match declared state: + {{ _multica_verify_daemon.stdout }} + quiet: true + +- name: Verify | Every selected agent CLI is registered as an online runtime + ansible.builtin.command: + argv: "{{ _multica_cmd + ['runtime', 'list', '--output', 'json'] }}" + become: true + become_user: "{{ multica_user }}" + environment: "{{ _multica_user_env }}" + register: _multica_verify_runtimes + until: + - _multica_verify_runtimes.rc == 0 + - _multica_expected_providers + | difference(_multica_verify_runtimes.stdout | from_json + | selectattr('daemon_id', 'equalto', (_multica_verify_daemon.stdout | from_json).daemon_id) + | selectattr('status', 'equalto', 'online') + | map(attribute='provider') | list) + | length == 0 + retries: "{{ multica_verify_retries }}" + delay: "{{ multica_verify_delay }}" + changed_when: false + check_mode: false + vars: + _multica_expected_providers: >- + {{ multica_agent_clis | map('extract', multica_agent_cli_catalog, 'provider') | list }} + +- name: Verify | Runtimes name this host + ansible.builtin.assert: + that: + - _multica_verify_runtimes.stdout | from_json + | selectattr('daemon_id', 'equalto', (_multica_verify_daemon.stdout | from_json).daemon_id) + | rejectattr('device_info', 'search', '^' ~ (multica_device_name | regex_escape)) + | list | length == 0 + fail_msg: >- + Runtimes registered under this daemon do not report device_name + "{{ multica_device_name }}". + quiet: true + +- name: Verify | Stat the workspaces root + ansible.builtin.stat: + path: "{{ multica_workspaces_root }}" + register: _multica_verify_ws + become: true + +- name: Verify | Re-prove the daemon user can write to the workspaces root + ansible.builtin.command: + argv: + - /bin/sh + - -c + - 'd="$1"; f="$d/.multica-write-test.$$"; : > "$f" && rm -f "$f"' + - sh + - "{{ multica_workspaces_root }}" + become: true + become_user: "{{ multica_user }}" + register: _multica_verify_ws_write + changed_when: false + failed_when: false + check_mode: false + +- name: Verify | Workspaces root assertions + ansible.builtin.assert: + that: + - _multica_verify_ws.stat.exists + - _multica_verify_ws.stat.isdir + - _multica_verify_ws.stat.pw_name == multica_user + - _multica_verify_ws_write.rc == 0 + fail_msg: >- + {{ multica_workspaces_root }} is missing, not a directory, or not + owned/writable by {{ multica_user }}. + quiet: true + +- name: Verify | Workspaces root is on the expected mount + ansible.builtin.assert: + that: + - multica_workspaces_root_mount in (ansible_facts['mounts'] | map(attribute='mount') | list) + fail_msg: "{{ multica_workspaces_root_mount }} is not mounted." + quiet: true + when: multica_workspaces_root_mount | length > 0 + +- name: Verify | Report what still needs a human + ansible.builtin.debug: + msg: >- + {{ multica_device_name }} is provisioned: multica {{ multica_version }}, + daemon running, runtimes online for + {{ multica_agent_clis | join(', ') }}. + Remaining manual step(s): + {{ multica_agent_clis + | select('in', multica_agent_cli_catalog) + | map('extract', multica_agent_cli_catalog) + | selectattr('manual_login', 'defined') + | selectattr('manual_login') + | map(attribute='manual_login_hint') + | list | join('; ') | default('none', true) }} diff --git a/ansible/roles/multica_agent_host/tasks/workspaces_root.yml b/ansible/roles/multica_agent_host/tasks/workspaces_root.yml new file mode 100644 index 0000000..f53089b --- /dev/null +++ b/ansible/roles/multica_agent_host/tasks/workspaces_root.yml @@ -0,0 +1,41 @@ +--- +# Guard against the mount not being present: without it the daemon happily +# writes run workspaces onto the root filesystem and nobody notices until it +# fills up. +- name: Workspaces root | Assert it lands on the expected mount + ansible.builtin.assert: + that: + - multica_workspaces_root_mount in (ansible_facts['mounts'] | map(attribute='mount') | list) + - multica_workspaces_root == multica_workspaces_root_mount + or multica_workspaces_root.startswith(multica_workspaces_root_mount.rstrip('/') ~ '/') + fail_msg: >- + multica_workspaces_root ({{ multica_workspaces_root }}) is not on + {{ multica_workspaces_root_mount }}, which is not currently mounted or + does not contain it. Mount it before provisioning. + quiet: true + when: multica_workspaces_root_mount | length > 0 + +- name: Workspaces root | Create the directory + ansible.builtin.file: + path: "{{ multica_workspaces_root }}" + state: directory + owner: "{{ multica_user }}" + group: "{{ multica_group }}" + mode: "{{ multica_workspaces_root_mode }}" + become: true + +# Ownership is not writability: a read-only or root-squashed network mount +# passes the file module and fails the first run. +- name: Workspaces root | Prove the daemon user can write to it + ansible.builtin.command: + argv: + - /bin/sh + - -c + - 'd="$1"; f="$d/.multica-write-test.$$"; : > "$f" && rm -f "$f"' + - sh + - "{{ multica_workspaces_root }}" + become: true + become_user: "{{ multica_user }}" + changed_when: false + register: _multica_ws_write + failed_when: _multica_ws_write.rc != 0 diff --git a/ansible/roles/multica_agent_host/vars/main.yml b/ansible/roles/multica_agent_host/vars/main.yml new file mode 100644 index 0000000..e3963cb --- /dev/null +++ b/ansible/roles/multica_agent_host/vars/main.yml @@ -0,0 +1,16 @@ +--- +# Internal, not meant to be overridden. + +# ansible_architecture -> release asset arch +_multica_arch_map: # noqa: var-naming[no-role-prefix] + x86_64: amd64 + amd64: amd64 + aarch64: arm64 + arm64: arm64 + +# ansible_architecture -> nodejs.org dist arch +_multica_node_arch_map: # noqa: var-naming[no-role-prefix] + x86_64: x64 + amd64: x64 + aarch64: arm64 + arm64: arm64 diff --git a/ansible/tests/expressions.yml b/ansible/tests/expressions.yml new file mode 100644 index 0000000..9f8a9de --- /dev/null +++ b/ansible/tests/expressions.yml @@ -0,0 +1,275 @@ +--- +# Offline regression tests for the role's Jinja expressions, run against +# fixtures captured from a real `multica` CLI. No target host required: +# +# ansible-playbook -i localhost, tests/expressions.yml +# +# This does NOT replace the scratch-VM run described in the role README; it +# catches the class of bug where a filter chain silently returns the wrong +# type or quietly matches nothing. +- name: Expressions | parsing real multica CLI output + hosts: localhost + gather_facts: false + vars: + multica_server_url: "https://multica.wdw-labs.cloud" + multica_app_url: "" + multica_workspaces_root: "/srv/multica/workspaces" + multica_device_name: "rblundon--Mac16" + multica_runtime_name: "" + multica_max_concurrent_tasks: "" + multica_poll_interval: "" + multica_heartbeat_interval: "" + multica_agent_timeout: "" + multica_disable_auto_update: true + multica_version: "0.6.5" + multica_agent_clis: [claude, hermes] + multica_agent_cli_catalog: + claude: {provider: claude, method: npm} + hermes: {provider: hermes, method: script} + cursor: {provider: cursor, method: npm} + _multica_arch_map: {x86_64: amd64, aarch64: arm64} + + tasks: + - name: Load fixtures + ansible.builtin.set_fact: + rt: "{{ lookup('file', '{{ playbook_dir }}/fixtures/runtime-list.json') }}" + ds: "{{ lookup('file', '{{ playbook_dir }}/fixtures/daemon-status.json') }}" + cs_lines: "{{ lookup('file', '{{ playbook_dir }}/fixtures/config-show.txt').splitlines() }}" + ver_out: "multica 0.6.5 (commit: 654b8ca95, built: 2026-10-21T09:41:38Z)" + + # --- CLI version extraction (multica_cli.yml) --------------------------- + - name: Extract installed CLI version + ansible.builtin.set_fact: + found: >- + {{ ((([ver_out] | first | default('')).split() + ['', ''])[1]) }} + - name: Assert version extraction + ansible.builtin.assert: + that: [(found | trim) == multica_version] + success_msg: "CLI version extraction OK -> {{ found }}" + + # --- config show parsing (config.yml) ----------------------------------- + - name: Parse config show + ansible.builtin.set_fact: + current: >- + {{ dict( + cs_lines + | select('match', '^[a-z_]+:') + | map('regex_replace', '^([a-z_]+):\s*(.*?)\s*$', '\1|\2') + | map('split', '|') + ) }} + - name: Build desired config + ansible.builtin.set_fact: + desired: >- + {{ { + 'server_url': multica_server_url, + 'app_url': multica_app_url, + 'workspaces_root': multica_workspaces_root, + 'device_name': multica_device_name, + 'runtime_name': multica_runtime_name, + 'max_concurrent_tasks': multica_max_concurrent_tasks, + 'poll_interval': multica_poll_interval, + 'heartbeat_interval': multica_heartbeat_interval, + 'agent_timeout': multica_agent_timeout, + 'disable_auto_update': ('true' if multica_disable_auto_update | bool else 'false'), + } | dict2items + | rejectattr('value', 'equalto', '') + | list | items2dict }} + - name: Show parse results + ansible.builtin.debug: + msg: + parsed_keys: "{{ current.keys() | list }}" + disable_auto_update_now: "{{ current['disable_auto_update'] | default('MISSING') }}" + server_url_now: "{{ current['server_url'] | default('MISSING') }}" + desired: "{{ desired }}" + would_change: >- + {{ desired | dict2items + | selectattr('key', 'in', current.keys() | list) + | rejectattr('value', 'equalto', 'IMPOSSIBLE') + | map(attribute='key') | list }} + - name: Assert config parsing found the expected keys + ansible.builtin.assert: + that: + - "'server_url' in current" + - "'disable_auto_update' in current" + - current['server_url'] == '(not set)' + - current['disable_auto_update'] == 'false' + - desired | length == 4 + - "'app_url' not in desired" + success_msg: "config show parsing + desired-config filtering OK" + + # --- runtime verification (verify.yml) ---------------------------------- + - name: Compute online providers for this daemon + ansible.builtin.set_fact: + expected: "{{ multica_agent_clis | map('extract', multica_agent_cli_catalog, 'provider') | list }}" + online: >- + {{ rt | from_json + | selectattr('daemon_id', 'equalto', (ds | from_json).daemon_id) + | selectattr('status', 'equalto', 'online') + | map(attribute='provider') | list }} + - name: Assert every expected provider is online + ansible.builtin.assert: + that: [(expected | difference(online)) | length == 0] + success_msg: "runtime check OK: expected={{ expected }} online={{ online }}" + + - name: Assert device_info naming check + ansible.builtin.assert: + that: + - rt | from_json + | selectattr('daemon_id', 'equalto', (ds | from_json).daemon_id) + | rejectattr('device_info', 'search', '^' ~ (multica_device_name | regex_escape)) + | list | length == 0 + success_msg: "device_info check OK for {{ multica_device_name }}" + + - name: Assert daemon status fields + ansible.builtin.assert: + that: + - (ds | from_json).status == 'running' + - (ds | from_json).cli_version == multica_version + - ((ds | from_json).server_url | regex_replace('/$','')) == (multica_server_url | regex_replace('/$','')) + success_msg: "daemon status assertions OK" + + # --- negative case: a provider that is NOT online ------------------------ + - name: Assert a missing provider is detected + ansible.builtin.assert: + that: + - (['codex'] | difference(online)) | length == 1 + success_msg: "missing-provider detection OK" + +- name: Expressions | preflight fact-building + hosts: localhost + gather_facts: false + vars: + multica_install_dir: /usr/local/bin + multica_agent_cli_catalog: + claude: {provider: claude, method: npm} + hermes: {provider: hermes, method: script} + multica_release_download_base_url: "https://github.com/multica-ai/multica/releases/download" + multica_version: "0.6.5" + _multica_arch_map: {x86_64: amd64, aarch64: arm64} + tasks: + - name: Case - profile unset, npm CLI selected + ansible.builtin.set_fact: + multica_profile: "" + multica_agent_clis: [claude, hermes] + _multica_arch: "{{ _multica_arch_map['x86_64'] }}" + - name: Build facts + ansible.builtin.set_fact: + _multica_cmd: >- + {{ [multica_install_dir ~ '/multica'] + + (['--profile', multica_profile] if multica_profile | length > 0 else []) }} + _multica_node_required: >- + {{ multica_agent_clis + | map('extract', multica_agent_cli_catalog, 'method') + | select('equalto', 'npm') | list | length > 0 }} + _multica_release_url: >- + {{ multica_release_download_base_url }}/v{{ multica_version }}/multica-cli-{{ multica_version }}-linux-{{ _multica_arch }}.tar.gz + - name: Assert types and values + ansible.builtin.assert: + that: + - _multica_cmd is sequence + - _multica_cmd is not string + - _multica_cmd == ['/usr/local/bin/multica'] + - (_multica_cmd + ['daemon', 'status']) == ['/usr/local/bin/multica', 'daemon', 'status'] + - _multica_node_required | bool + - _multica_release_url == 'https://github.com/multica-ai/multica/releases/download/v0.6.5/multica-cli-0.6.5-linux-amd64.tar.gz' + success_msg: "argv prefix + node detection + URL OK: {{ _multica_cmd }}" + + - name: Case - profile set, script-only CLI selected + ansible.builtin.set_fact: + multica_profile: "haunted" + multica_agent_clis: [hermes] + - name: Rebuild facts + ansible.builtin.set_fact: + _multica_cmd: >- + {{ [multica_install_dir ~ '/multica'] + + (['--profile', multica_profile] if multica_profile | length > 0 else []) }} + _multica_node_required: >- + {{ multica_agent_clis + | map('extract', multica_agent_cli_catalog, 'method') + | select('equalto', 'npm') | list | length > 0 }} + - name: Assert profile + no-node case + ansible.builtin.assert: + that: + - _multica_cmd == ['/usr/local/bin/multica', '--profile', 'haunted'] + - not (_multica_node_required | bool) + success_msg: "profile prefix + node-not-required OK: {{ _multica_cmd }}" + + - name: Assert PAT prefix validation accepts both token kinds and rejects junk + ansible.builtin.assert: + that: + - "'mul_abc123' is match('^(mul_|mcn_)')" + - "'mcn_abc123' is match('^(mul_|mcn_)')" + - "not ('ghp_abc123' is match('^(mul_|mcn_)'))" + - "not ('' is match('^(mul_|mcn_)'))" + success_msg: "PAT validation OK" + + - name: Assert unknown-agent detection + ansible.builtin.assert: + that: + - "['claude', 'bogus'] | difference(multica_agent_cli_catalog.keys() | list) == ['bogus']" + - "['claude', 'hermes'] | difference(multica_agent_cli_catalog.keys() | list) | length == 0" + success_msg: "catalog validation OK" + + - name: Assert workspaces-root-under-mount logic + vars: + root: "/mnt/utilidor/multica/workspaces" + mnt: "/mnt/utilidor" + bad_root: "/mnt/utilidor-other/workspaces" + ansible.builtin.assert: + that: + - root == mnt or root.startswith(mnt.rstrip('/') ~ '/') + - not (bad_root == mnt or bad_root.startswith(mnt.rstrip('/') ~ '/')) + success_msg: "mount containment check OK (rejects /mnt/utilidor-other)" + +- name: Expressions | pinned agent-CLI version guard + hosts: localhost + gather_facts: false + vars: + # Mirrors the expression in tasks/agent_cli/npm.yml. + guard: >- + {{ rc != 0 + or (pin | default('') | length > 0 + and not ((out | default('')) is search( + '(^|[^0-9.])' ~ (pin | regex_escape) ~ '([^0-9.]|$)'))) }} + tasks: + - name: Pinned version already installed - must NOT reinstall + ansible.builtin.assert: + that: [not (guard | bool)] + success_msg: "2.1.304 pin vs '2.1.304 (Claude Code)' -> no install" + vars: {rc: 0, pin: "2.1.304", out: "2.1.304 (Claude Code)"} + + - name: Different version installed - must reinstall + ansible.builtin.assert: + that: [guard | bool] + success_msg: "2.1.304 pin vs 2.1.306 installed -> install" + vars: {rc: 0, pin: "2.1.304", out: "2.1.306 (Claude Code)"} + + - name: Longer version must not satisfy a shorter pin + ansible.builtin.assert: + that: [guard | bool] + success_msg: "2.1.304 pin vs 2.1.3045 installed -> install (no substring false-match)" + vars: {rc: 0, pin: "2.1.304", out: "2.1.3045 (Claude Code)"} + + - name: Prefix of installed version must not satisfy the pin + ansible.builtin.assert: + that: [guard | bool] + success_msg: "2.1.30 pin vs 2.1.304 installed -> install" + vars: {rc: 0, pin: "2.1.30", out: "2.1.304 (Claude Code)"} + + - name: Version embedded mid-string still matches (hermes-style output) + ansible.builtin.assert: + that: [not (guard | bool)] + success_msg: "0.21.5 pin vs 'Hermes Agent v0.21.5+2024.gb609c74' -> no install" + vars: {rc: 0, pin: "0.21.5", out: "Hermes Agent v0.21.5+2024.gb609c74 (2026.9.24)"} + + - name: Binary absent - must install regardless of pin + ansible.builtin.assert: + that: [guard | bool] + success_msg: "rc!=0 -> install" + vars: {rc: 1, pin: "2.1.304", out: ""} + + - name: Unpinned tracks latest - never reinstalls once present + ansible.builtin.assert: + that: [not (guard | bool)] + success_msg: "empty pin + present -> no install" + vars: {rc: 0, pin: "", out: "2.1.306 (Claude Code)"} diff --git a/ansible/tests/fixtures/config-show.txt b/ansible/tests/fixtures/config-show.txt new file mode 100644 index 0000000..5561ac8 --- /dev/null +++ b/ansible/tests/fixtures/config-show.txt @@ -0,0 +1,17 @@ +Config file: /home/multica/.config/multica/config.json +server_url: (not set) +app_url: (not set) +workspace_id: (not set) +device_name: (not set) +runtime_name: (not set) +workspaces_root: (not set) +max_concurrent_tasks: (not set) +poll_interval: (not set) +ws_claim_poll_interval: (not set) +heartbeat_interval: (not set) +agent_timeout: (not set) +codex_semantic_inactivity_timeout: (not set) +codex_handshake_timeout: (not set) +disable_auto_update: false +auto_update_check_interval: (not set) +disable_auto_reload: false diff --git a/ansible/tests/fixtures/daemon-status.json b/ansible/tests/fixtures/daemon-status.json new file mode 100644 index 0000000..b9fecca --- /dev/null +++ b/ansible/tests/fixtures/daemon-status.json @@ -0,0 +1,37 @@ +{ + "active_task_count": 1, + "agents": [ + "claude", + "opencode", + "hermes", + "cursor" + ], + "cli_commit": "654b8ca954034f8181eb6bbb4fc298dede760156", + "cli_version": "0.6.5", + "daemon_id": "01a15ad9-f27e-72da-a1a5-53ec15b53f9d", + "device_name": "rblundon--Mac16", + "failed_terminal_report_bytes": 0, + "failed_terminal_report_count": 0, + "launched_by": "desktop", + "os": "darwin", + "pending_terminal_report_bytes": 0, + "pending_terminal_report_count": 0, + "pid": 34832, + "profile": "desktop-multica.wdw-labs.cloud", + "resource_wait_task_count": 0, + "running_task_count": 1, + "server_url": "https://multica.wdw-labs.cloud", + "status": "running", + "uptime": "4h12m4s", + "workspaces": [ + { + "id": "00000000-0000-0000-0000-000000000001", + "runtimes": [ + "ea210ad4-b0e8-4663-a212-7d6878668ec7", + "eb68ea3f-78d6-4214-974d-9022aff55216", + "245f271d-339b-47d5-8ea6-1e7eb6576ece", + "c8316de7-5668-4276-8b31-3e7f873b8499" + ] + } + ] +} diff --git a/ansible/tests/fixtures/runtime-list.json b/ansible/tests/fixtures/runtime-list.json new file mode 100644 index 0000000..dd2ec01 --- /dev/null +++ b/ansible/tests/fixtures/runtime-list.json @@ -0,0 +1,146 @@ +[ + { + "created_at": "2026-10-20T19:56:21Z", + "custom_name": null, + "daemon_id": "01a15ad9-f27e-72da-a1a5-53ec15b53f9d", + "device_info": "rblundon--Mac16 \u00b7 2.1.274 (Claude Code)", + "id": "ea210ad4-b0e8-4663-a212-7d6878668ec7", + "last_seen_at": "2026-10-21T19:47:35Z", + "launch_header": "claude (stream-json)", + "metadata": { + "capabilities": [ + "skill-bundles-v1", + "coalesced-comments-v1", + "execution-manifest-v1", + "agent-skill-v1", + "remote-mcp-v1", + "local-worktree-v1", + "source_context_quick_create_v1", + "rpc-v1", + "platform-skill-v1", + "checkout-keeps-work-v1", + "joined-wakeups-v1" + ], + "cli_version": "0.6.5", + "launched_by": "desktop", + "version": "2.1.274 (Claude Code)" + }, + "name": "Claude (rblundon--Mac16)", + "owner_id": "00000000-0000-0000-0000-000000000000", + "profile_id": null, + "provider": "claude", + "runtime_mode": "local", + "status": "online", + "updated_at": "2026-10-21T17:09:10Z", + "visibility": "private", + "workspace_id": "00000000-0000-0000-0000-000000000001" + }, + { + "created_at": "2026-10-20T19:56:22Z", + "custom_name": null, + "daemon_id": "01a15ad9-f27e-72da-a1a5-53ec15b53f9d", + "device_info": "rblundon--Mac16 \u00b7 2025.09.18-7ae6800", + "id": "eb68ea3f-78d6-4214-974d-9022aff55216", + "last_seen_at": "2026-10-21T19:47:35Z", + "launch_header": "cursor-agent (stream-json)", + "metadata": { + "capabilities": [ + "skill-bundles-v1", + "coalesced-comments-v1", + "execution-manifest-v1", + "agent-skill-v1", + "remote-mcp-v1", + "local-worktree-v1", + "source_context_quick_create_v1", + "rpc-v1", + "platform-skill-v1", + "checkout-keeps-work-v1", + "joined-wakeups-v1" + ], + "cli_version": "0.6.5", + "launched_by": "desktop", + "version": "2025.09.18-7ae6800" + }, + "name": "Cursor (rblundon--Mac16)", + "owner_id": "00000000-0000-0000-0000-000000000000", + "profile_id": null, + "provider": "cursor", + "runtime_mode": "local", + "status": "online", + "updated_at": "2026-10-21T17:09:10Z", + "visibility": "private", + "workspace_id": "00000000-0000-0000-0000-000000000001" + }, + { + "created_at": "2026-10-20T19:56:22Z", + "custom_name": null, + "daemon_id": "01a15ad9-f27e-72da-a1a5-53ec15b53f9d", + "device_info": "rblundon--Mac16 \u00b7 Hermes Agent v0.21.5+2024.gb609c74 (2026.9.24) \u00b7 upstream f1247d2e", + "id": "245f271d-339b-47d5-8ea6-1e7eb6576ece", + "last_seen_at": "2026-10-21T19:47:35Z", + "launch_header": "hermes acp", + "metadata": { + "capabilities": [ + "skill-bundles-v1", + "coalesced-comments-v1", + "execution-manifest-v1", + "agent-skill-v1", + "remote-mcp-v1", + "local-worktree-v1", + "source_context_quick_create_v1", + "rpc-v1", + "platform-skill-v1", + "checkout-keeps-work-v1", + "joined-wakeups-v1" + ], + "cli_version": "0.6.5", + "launched_by": "desktop", + "version": "Hermes Agent v0.21.5+2024.gb609c74 (2026.9.24) \u00b7 upstream f1247d2e" + }, + "name": "Hermes (rblundon--Mac16)", + "owner_id": "00000000-0000-0000-0000-000000000000", + "profile_id": null, + "provider": "hermes", + "runtime_mode": "local", + "status": "online", + "updated_at": "2026-10-21T17:09:10Z", + "visibility": "private", + "workspace_id": "00000000-0000-0000-0000-000000000001" + }, + { + "created_at": "2026-10-20T19:56:22Z", + "custom_name": null, + "daemon_id": "01a15ad9-f27e-72da-a1a5-53ec15b53f9d", + "device_info": "rblundon--Mac16 \u00b7 1.18.20", + "id": "c8316de7-5668-4276-8b31-3e7f873b8499", + "last_seen_at": "2026-10-21T19:47:35Z", + "launch_header": "opencode run (json)", + "metadata": { + "capabilities": [ + "skill-bundles-v1", + "coalesced-comments-v1", + "execution-manifest-v1", + "agent-skill-v1", + "remote-mcp-v1", + "local-worktree-v1", + "source_context_quick_create_v1", + "rpc-v1", + "platform-skill-v1", + "checkout-keeps-work-v1", + "joined-wakeups-v1" + ], + "cli_version": "0.6.5", + "launched_by": "desktop", + "version": "1.18.20" + }, + "name": "Opencode (rblundon--Mac16)", + "owner_id": "00000000-0000-0000-0000-000000000000", + "profile_id": null, + "provider": "opencode", + "runtime_mode": "local", + "status": "online", + "updated_at": "2026-10-21T17:09:10Z", + "visibility": "private", + "workspace_id": "00000000-0000-0000-0000-000000000001" + } +] diff --git a/fastpass/argocd/apps-of-apps.yaml b/fastpass/argocd/apps-of-apps.yaml new file mode 100644 index 0000000..b96f592 --- /dev/null +++ b/fastpass/argocd/apps-of-apps.yaml @@ -0,0 +1,45 @@ +# ------------------------------------------------------------------------------ +# App of Apps — Root +# +# This is applied ONCE after ArgoCD is bootstrapped. It is the second and +# final manual kubectl apply: +# +# kubectl apply -n argocd -f cluster/argocd/apps-of-apps.yaml +# +# From this point forward, all changes are made via git. ArgoCD self-manages. +# +# This Application watches cluster/platform/ and cluster/applications/ and +# creates child Applications for everything it finds there. +# ------------------------------------------------------------------------------ +apiVersion: argoproj.io/v1alpha1 +kind: Application +metadata: + name: apps-of-apps + namespace: argocd + annotations: + # Wave 0 — this is the root, everything else comes after + argocd.argoproj.io/sync-wave: "0" +spec: + project: default + source: + repoURL: https://mad-tea-party.wdw-labs.cloud/rblundon/homelab.git + targetRevision: main + path: fastpass + directory: + recurse: true + include: "*/application.yaml" + destination: + server: https://kubernetes.default.svc + namespace: argocd + syncPolicy: + automated: + prune: true + selfHeal: true + syncOptions: + - CreateNamespace=true + - ServerSideApply=true + ignoreDifferences: + - group: argoproj.io + kind: Application + jsonPointers: + - /spec/syncPolicy/automated diff --git a/fastpass/argocd/externalsecret.yaml b/fastpass/argocd/externalsecret.yaml new file mode 100644 index 0000000..1ff02be --- /dev/null +++ b/fastpass/argocd/externalsecret.yaml @@ -0,0 +1,29 @@ +# ------------------------------------------------------------------------------ +# ExternalSecret — Authentik OIDC credentials for ArgoCD +# +# Targets argocd-secret directly — ArgoCD reads oidc.authentik.* keys +# from its own secret. ESO merges these fields into the existing secret +# without overwriting admin.password etc. (creationPolicy: Merge) +# ------------------------------------------------------------------------------ +apiVersion: external-secrets.io/v1beta1 +kind: ExternalSecret +metadata: + name: argocd-oidc-secret + namespace: argocd +spec: + secretStoreRef: + kind: ClusterSecretStore + name: onepassword-connect + refreshInterval: "1h" + target: + name: argocd-secret + creationPolicy: Merge + data: + - secretKey: oidc.authentik.clientID + remoteRef: + key: argocd-oidc + property: client-id + - secretKey: oidc.authentik.clientSecret + remoteRef: + key: argocd-oidc + property: client-secret diff --git a/fastpass/argocd/install.yaml b/fastpass/argocd/install.yaml new file mode 100644 index 0000000..f3d57c9 --- /dev/null +++ b/fastpass/argocd/install.yaml @@ -0,0 +1,21 @@ +# ------------------------------------------------------------------------------ +# ArgoCD Install — v3.4.2 +# +# This is NOT applied by ArgoCD itself (chicken-and-egg). +# It is the ONE manual bootstrap step: +# +# kubectl create namespace argocd +# kubectl apply -n argocd --server-side --force-conflicts -f install.yaml +# +# After this, everything else is driven by apps-of-apps.yaml. +# +# To upgrade ArgoCD: update the version below and re-apply. +# --server-side --force-conflicts is required due to CRD size limits. +# +# Source: https://github.com/argoproj/argo-cd/releases/tag/v3.4.2 +# ------------------------------------------------------------------------------ +# kubectl apply -n argocd --server-side --force-conflicts -f \ +# https://raw.githubusercontent.com/argoproj/argo-cd/v3.4.2/manifests/install.yaml +# +# We reference the upstream manifest directly rather than vendoring it. +# Pin upgrades by changing the version tag above and re-running the command. diff --git a/fastpass/argocd/namespace.yaml b/fastpass/argocd/namespace.yaml new file mode 100644 index 0000000..a040f2b --- /dev/null +++ b/fastpass/argocd/namespace.yaml @@ -0,0 +1,4 @@ +apiVersion: v1 +kind: Namespace +metadata: + name: argocd diff --git a/fastpass/argocd/values.yaml b/fastpass/argocd/values.yaml new file mode 100644 index 0000000..5360122 --- /dev/null +++ b/fastpass/argocd/values.yaml @@ -0,0 +1,135 @@ +# ─── ArgoCD Helm values — fastpass cluster ───────────────────────────────────── +# Chart: argo/argo-cd +# Version: 7.8.23 +# +# Install: +# helm repo add argo https://argoproj.github.io/argo-helm +# helm repo update +# helm upgrade --install argocd argo/argo-cd \ +# --namespace argocd \ +# --create-namespace \ +# --version 7.8.23 \ +# --values cluster/argocd/values.yaml \ +# --wait +# +# Then apply the root app-of-apps (second and final manual step): +# kubectl apply -n argocd -f cluster/argocd/apps-of-apps.yaml +# ─────────────────────────────────────────────────────────────────────────────── + +# ─── Global ───────────────────────────────────────────────────────────────────── +global: + domain: carousel-of-progress.wdw-labs.cloud + +# ─── ArgoCD server ────────────────────────────────────────────────────────────── +server: + # Run insecure — TLS is terminated at ingress-nginx, not at ArgoCD itself. + # Without this, ArgoCD redirects to HTTPS internally and breaks behind ingress. + extraArgs: + - --insecure + metrics: + enabled: true + serviceMonitor: + enabled: true + ingress: + enabled: true + ingressClassName: nginx + hostname: carousel-of-progress.wdw-labs.cloud + tls: true + annotations: + cert-manager.io/cluster-issuer: letsencrypt-prod + nginx.ingress.kubernetes.io/ssl-passthrough: "false" + nginx.ingress.kubernetes.io/backend-protocol: "HTTP" + +# ─── ConfigMap — core ArgoCD settings ─────────────────────────────────────────── +configs: + params: + server.insecure: "true" + server.session.duration: "720h" + + cm: + url: https://carousel-of-progress.wdw-labs.cloud + + # Authentik native OIDC SSO + # Client secret pulled from 1Password via ESO → argocd-oidc-secret + oidc.config: | + name: Authentik + issuer: https://authentik.local.mk-labs.cloud/application/o/argocd/ + clientID: $oidc.authentik.clientID + clientSecret: $oidc.authentik.clientSecret + requestedScopes: + - openid + - profile + - email + - groups + groupsClaim: groups + + # Resource health checks — ignore progressing Cilium CRDs + resource.customizations.health.cilium.io_CiliumLoadBalancerIPPool: | + hs = {} + hs.status = "Healthy" + hs.message = "IP pool ready" + return hs + + # Ignore ESO-injected default fields on ExternalSecret resources. + resource.customizations.ignoreDifferences.external-secrets.io_ExternalSecret: | + jqPathExpressions: + - '.spec.data[]?.remoteRef?.conversionStrategy' + - '.spec.data[]?.remoteRef?.decodingStrategy' + - '.spec.data[]?.remoteRef?.metadataPolicy' + - '.spec.dataFrom[]?.extract?.conversionStrategy' + - '.spec.dataFrom[]?.extract?.decodingStrategy' + - '.spec.dataFrom[]?.extract?.metadataPolicy' + + # RBAC — argocd-admins group from Authentik gets full access + rbac: + policy.default: role:readonly + policy.csv: | + g, argocd-admins, role:admin + g, argocd-viewers, role:readonly + + # Repository credentials — Gitea SSH + repositories: + homelab-repo: + url: git@gitea.mk-labs.cloud:rblundon/homelab.git + name: homelab + type: git + insecure: "false" + +# ─── Redis ────────────────────────────────────────────────────────────────────── +redis: + enabled: true + +# ─── Notifications ────────────────────────────────────────────────────────────── +# Disabled for now — wire into n8n via webhook later +notifications: + enabled: false + +# ─── ApplicationSet controller ────────────────────────────────────────────────── +applicationSet: + enabled: true + metrics: + enabled: true + serviceMonitor: + enabled: true + +# ─── Dex ──────────────────────────────────────────────────────────────────────── +# Disabled — using Authentik as the native OIDC provider +dex: + enabled: false + +# Disable default admin user +admin: + enabled: false + +# Enable Prometheus metrics for ArgoCD +controller: + metrics: + enabled: true + serviceMonitor: + enabled: true + +repoServer: + metrics: + enabled: true + serviceMonitor: + enabled: true