posts.py 133 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957958959960961962963964965966967968969970971972973974975976977978979980981982983984985986987988989990991992993994995996997998999100010011002100310041005100610071008100910101011101210131014101510161017101810191020102110221023102410251026102710281029103010311032103310341035103610371038103910401041104210431044104510461047104810491050105110521053105410551056105710581059106010611062106310641065106610671068106910701071107210731074107510761077107810791080108110821083108410851086108710881089109010911092109310941095109610971098109911001101110211031104110511061107110811091110111111121113111411151116111711181119112011211122112311241125112611271128112911301131113211331134113511361137113811391140114111421143114411451146114711481149115011511152115311541155115611571158115911601161116211631164116511661167116811691170117111721173117411751176117711781179118011811182118311841185118611871188118911901191119211931194119511961197119811991200120112021203120412051206120712081209121012111212121312141215121612171218121912201221122212231224122512261227122812291230123112321233123412351236123712381239124012411242124312441245124612471248124912501251125212531254125512561257125812591260126112621263126412651266126712681269127012711272127312741275127612771278127912801281128212831284128512861287128812891290129112921293129412951296129712981299130013011302130313041305130613071308130913101311131213131314131513161317131813191320132113221323132413251326132713281329133013311332133313341335133613371338133913401341134213431344134513461347134813491350135113521353135413551356135713581359136013611362136313641365136613671368136913701371137213731374137513761377137813791380138113821383138413851386138713881389139013911392139313941395139613971398139914001401140214031404140514061407140814091410141114121413141414151416141714181419142014211422142314241425142614271428142914301431143214331434143514361437143814391440144114421443144414451446144714481449145014511452145314541455145614571458145914601461146214631464146514661467146814691470147114721473147414751476147714781479148014811482148314841485148614871488148914901491149214931494149514961497149814991500150115021503150415051506150715081509151015111512151315141515151615171518151915201521152215231524152515261527152815291530153115321533153415351536153715381539154015411542154315441545154615471548154915501551155215531554155515561557155815591560156115621563156415651566156715681569157015711572157315741575157615771578157915801581158215831584158515861587158815891590159115921593159415951596159715981599160016011602160316041605160616071608160916101611161216131614161516161617161816191620162116221623162416251626162716281629163016311632163316341635163616371638163916401641164216431644164516461647164816491650165116521653165416551656165716581659166016611662166316641665166616671668166916701671167216731674167516761677167816791680168116821683168416851686168716881689169016911692169316941695169616971698169917001701170217031704170517061707170817091710171117121713171417151716171717181719172017211722172317241725172617271728172917301731173217331734173517361737173817391740174117421743174417451746174717481749175017511752175317541755175617571758175917601761176217631764176517661767176817691770177117721773177417751776177717781779178017811782178317841785178617871788178917901791179217931794179517961797179817991800180118021803180418051806180718081809181018111812181318141815181618171818181918201821182218231824182518261827182818291830183118321833183418351836183718381839184018411842184318441845184618471848184918501851185218531854185518561857185818591860186118621863186418651866186718681869187018711872187318741875187618771878187918801881188218831884188518861887188818891890189118921893189418951896189718981899190019011902190319041905190619071908190919101911191219131914191519161917191819191920192119221923192419251926192719281929193019311932193319341935193619371938193919401941194219431944194519461947194819491950195119521953195419551956195719581959196019611962196319641965196619671968196919701971197219731974197519761977197819791980198119821983198419851986198719881989199019911992199319941995199619971998199920002001200220032004200520062007200820092010201120122013201420152016201720182019202020212022202320242025202620272028202920302031203220332034203520362037203820392040204120422043204420452046204720482049205020512052205320542055205620572058205920602061206220632064206520662067206820692070207120722073207420752076207720782079208020812082208320842085208620872088208920902091209220932094209520962097209820992100210121022103210421052106210721082109211021112112211321142115211621172118211921202121212221232124212521262127212821292130213121322133213421352136213721382139214021412142214321442145214621472148214921502151215221532154215521562157215821592160216121622163216421652166216721682169217021712172217321742175217621772178217921802181218221832184218521862187218821892190219121922193219421952196219721982199220022012202220322042205220622072208220922102211221222132214221522162217221822192220222122222223222422252226222722282229223022312232223322342235223622372238223922402241224222432244224522462247224822492250225122522253225422552256225722582259226022612262226322642265226622672268226922702271227222732274227522762277227822792280228122822283228422852286228722882289229022912292229322942295229622972298229923002301230223032304230523062307230823092310231123122313231423152316231723182319232023212322232323242325232623272328232923302331233223332334233523362337233823392340234123422343234423452346234723482349235023512352235323542355235623572358235923602361236223632364236523662367236823692370237123722373237423752376237723782379238023812382238323842385238623872388238923902391239223932394239523962397239823992400240124022403240424052406240724082409241024112412241324142415241624172418241924202421242224232424242524262427242824292430243124322433243424352436243724382439244024412442244324442445244624472448244924502451245224532454245524562457245824592460246124622463246424652466246724682469247024712472247324742475247624772478247924802481248224832484248524862487248824892490249124922493249424952496249724982499250025012502250325042505250625072508250925102511251225132514251525162517251825192520252125222523252425252526252725282529253025312532253325342535253625372538253925402541254225432544254525462547254825492550255125522553255425552556255725582559256025612562256325642565256625672568256925702571257225732574257525762577257825792580258125822583258425852586258725882589259025912592259325942595259625972598259926002601260226032604260526062607260826092610261126122613261426152616261726182619262026212622262326242625262626272628262926302631263226332634263526362637263826392640264126422643264426452646264726482649265026512652265326542655265626572658265926602661266226632664266526662667266826692670267126722673267426752676267726782679268026812682268326842685268626872688268926902691269226932694269526962697269826992700270127022703270427052706270727082709271027112712271327142715271627172718271927202721272227232724272527262727272827292730273127322733273427352736273727382739274027412742274327442745274627472748274927502751275227532754275527562757275827592760276127622763276427652766276727682769277027712772277327742775277627772778277927802781278227832784278527862787278827892790279127922793279427952796279727982799280028012802280328042805280628072808280928102811281228132814281528162817281828192820282128222823282428252826282728282829283028312832283328342835283628372838283928402841284228432844284528462847284828492850285128522853285428552856285728582859286028612862286328642865286628672868286928702871287228732874287528762877287828792880288128822883288428852886288728882889289028912892289328942895289628972898289929002901290229032904290529062907290829092910291129122913291429152916291729182919292029212922292329242925292629272928292929302931293229332934293529362937293829392940294129422943294429452946294729482949295029512952295329542955295629572958295929602961296229632964296529662967296829692970297129722973297429752976297729782979298029812982298329842985298629872988298929902991299229932994299529962997299829993000300130023003300430053006300730083009301030113012301330143015301630173018301930203021302230233024302530263027302830293030303130323033303430353036303730383039304030413042304330443045304630473048304930503051305230533054305530563057305830593060306130623063306430653066306730683069307030713072307330743075307630773078307930803081308230833084308530863087308830893090309130923093309430953096309730983099310031013102310331043105310631073108310931103111311231133114311531163117311831193120312131223123312431253126312731283129313031313132313331343135313631373138313931403141314231433144314531463147314831493150315131523153315431553156315731583159316031613162316331643165316631673168316931703171317231733174317531763177317831793180318131823183318431853186318731883189319031913192319331943195319631973198319932003201320232033204320532063207320832093210321132123213321432153216321732183219322032213222322332243225322632273228322932303231323232333234323532363237323832393240324132423243324432453246324732483249325032513252325332543255325632573258325932603261326232633264326532663267326832693270327132723273327432753276327732783279328032813282328332843285328632873288328932903291329232933294329532963297329832993300330133023303330433053306330733083309331033113312331333143315331633173318331933203321332233233324332533263327332833293330333133323333333433353336333733383339334033413342334333443345334633473348334933503351335233533354335533563357335833593360336133623363336433653366336733683369337033713372337333743375337633773378337933803381338233833384338533863387338833893390339133923393339433953396339733983399340034013402340334043405340634073408340934103411341234133414341534163417341834193420342134223423342434253426342734283429343034313432343334343435343634373438343934403441344234433444344534463447344834493450345134523453345434553456345734583459346034613462346334643465346634673468346934703471347234733474347534763477347834793480348134823483348434853486348734883489349034913492349334943495349634973498349935003501350235033504350535063507350835093510351135123513351435153516351735183519352035213522352335243525352635273528352935303531
  1. __filename__ = "posts.py"
  2. __author__ = "Bob Mottram"
  3. __license__ = "AGPL3+"
  4. __version__ = "1.1.0"
  5. __maintainer__ = "Bob Mottram"
  6. __email__ = "bob@freedombone.net"
  7. __status__ = "Production"
  8. import json
  9. import html
  10. import datetime
  11. import os
  12. import shutil
  13. import sys
  14. import time
  15. from socket import error as SocketError
  16. from time import gmtime, strftime
  17. from collections import OrderedDict
  18. from threads import threadWithTrace
  19. from cache import storePersonInCache
  20. from cache import getPersonFromCache
  21. from cache import expirePersonCache
  22. from pprint import pprint
  23. from session import createSession
  24. from session import getJson
  25. from session import postJson
  26. from session import postJsonString
  27. from session import postImage
  28. from webfinger import webfingerHandle
  29. from httpsig import createSignedHeader
  30. from utils import siteIsActive
  31. from utils import removePostFromCache
  32. from utils import getCachedPostFilename
  33. from utils import getStatusNumber
  34. from utils import createPersonDir
  35. from utils import urlPermitted
  36. from utils import getNicknameFromActor
  37. from utils import getDomainFromActor
  38. from utils import deletePost
  39. from utils import validNickname
  40. from utils import locatePost
  41. from utils import loadJson
  42. from utils import saveJson
  43. from capabilities import getOcapFilename
  44. from capabilities import capabilitiesUpdate
  45. from media import attachMedia
  46. from media import replaceYouTube
  47. from content import removeLongWords
  48. from content import addHtmlTags
  49. from content import replaceEmojiFromTags
  50. from content import removeTextFormatting
  51. from auth import createBasicAuthHeader
  52. from config import getConfigParam
  53. from blocking import isBlocked
  54. from filters import isFiltered
  55. from git import convertPostToPatch
  56. from jsonldsig import jsonldSign
  57. from petnames import resolvePetnames
  58. # try:
  59. # from BeautifulSoup import BeautifulSoup
  60. # except ImportError:
  61. # from bs4 import BeautifulSoup
  62. def isModerator(baseDir: str, nickname: str) -> bool:
  63. """Returns true if the given nickname is a moderator
  64. """
  65. moderatorsFile = baseDir + '/accounts/moderators.txt'
  66. if not os.path.isfile(moderatorsFile):
  67. if getConfigParam(baseDir, 'admin') == nickname:
  68. return True
  69. return False
  70. with open(moderatorsFile, "r") as f:
  71. lines = f.readlines()
  72. if len(lines) == 0:
  73. if getConfigParam(baseDir, 'admin') == nickname:
  74. return True
  75. for moderator in lines:
  76. moderator = moderator.strip('\n').strip('\r')
  77. if moderator == nickname:
  78. return True
  79. return False
  80. def noOfFollowersOnDomain(baseDir: str, handle: str,
  81. domain: str, followFile='followers.txt') -> int:
  82. """Returns the number of followers of the given handle from the given domain
  83. """
  84. filename = baseDir + '/accounts/' + handle + '/' + followFile
  85. if not os.path.isfile(filename):
  86. return 0
  87. ctr = 0
  88. with open(filename, "r") as followersFilename:
  89. for followerHandle in followersFilename:
  90. if '@' in followerHandle:
  91. followerDomain = followerHandle.split('@')[1]
  92. followerDomain = followerDomain.replace('\n', '')
  93. followerDomain = followerDomain.replace('\r', '')
  94. if domain == followerDomain:
  95. ctr += 1
  96. return ctr
  97. def getPersonKey(nickname: str, domain: str, baseDir: str, keyType='public',
  98. debug=False):
  99. """Returns the public or private key of a person
  100. """
  101. handle = nickname + '@' + domain
  102. keyFilename = baseDir + '/keys/' + keyType + '/' + handle.lower() + '.key'
  103. if not os.path.isfile(keyFilename):
  104. if debug:
  105. print('DEBUG: private key file not found: ' + keyFilename)
  106. return ''
  107. keyPem = ''
  108. with open(keyFilename, "r") as pemFile:
  109. keyPem = pemFile.read()
  110. if len(keyPem) < 20:
  111. if debug:
  112. print('DEBUG: private key was too short: ' + keyPem)
  113. return ''
  114. return keyPem
  115. def cleanHtml(rawHtml: str) -> str:
  116. # text=BeautifulSoup(rawHtml, 'html.parser').get_text()
  117. text = rawHtml
  118. return html.unescape(text)
  119. def getUserUrl(wfRequest: {}) -> str:
  120. if wfRequest.get('links'):
  121. for link in wfRequest['links']:
  122. if link.get('type') and link.get('href'):
  123. if link['type'] == 'application/activity+json':
  124. if not ('/users/' in link['href'] or
  125. '/profile/' in link['href'] or
  126. '/channel/' in link['href']):
  127. print('Webfinger activity+json contains ' +
  128. 'single user instance actor')
  129. return link['href']
  130. return None
  131. def parseUserFeed(session, feedUrl: str, asHeader: {},
  132. projectVersion: str, httpPrefix: str,
  133. domain: str, depth=0) -> {}:
  134. if depth > 10:
  135. return None
  136. feedJson = getJson(session, feedUrl, asHeader, None,
  137. projectVersion, httpPrefix, domain)
  138. if not feedJson:
  139. return None
  140. if 'orderedItems' in feedJson:
  141. for item in feedJson['orderedItems']:
  142. yield item
  143. nextUrl = None
  144. if 'first' in feedJson:
  145. nextUrl = feedJson['first']
  146. elif 'next' in feedJson:
  147. nextUrl = feedJson['next']
  148. if nextUrl:
  149. if isinstance(nextUrl, str):
  150. if '?max_id=0' not in nextUrl:
  151. userFeed = \
  152. parseUserFeed(session, nextUrl, asHeader,
  153. projectVersion, httpPrefix,
  154. domain, depth+1)
  155. if userFeed:
  156. for item in userFeed:
  157. yield item
  158. elif isinstance(nextUrl, dict):
  159. userFeed = nextUrl
  160. if userFeed.get('orderedItems'):
  161. for item in userFeed['orderedItems']:
  162. yield item
  163. def getPersonBox(baseDir: str, session, wfRequest: {},
  164. personCache: {},
  165. projectVersion: str, httpPrefix: str,
  166. nickname: str, domain: str,
  167. boxName='inbox') -> (str, str, str, str, str, str, str, str):
  168. profileStr = 'https://www.w3.org/ns/activitystreams'
  169. asHeader = {
  170. 'Accept': 'application/activity+json; profile="' + profileStr + '"'
  171. }
  172. if not wfRequest.get('errors'):
  173. personUrl = getUserUrl(wfRequest)
  174. else:
  175. if nickname == 'dev':
  176. # try single user instance
  177. print('getPersonBox: Trying single user instance with ld+json')
  178. personUrl = httpPrefix + '://' + domain
  179. asHeader = {
  180. 'Accept': 'application/ld+json; profile="' + profileStr + '"'
  181. }
  182. else:
  183. personUrl = httpPrefix + '://' + domain + '/users/' + nickname
  184. if not personUrl:
  185. return None, None, None, None, None, None, None, None
  186. personJson = getPersonFromCache(baseDir, personUrl, personCache)
  187. if not personJson:
  188. if '/channel/' in personUrl:
  189. asHeader = {
  190. 'Accept': 'application/ld+json; profile="' + profileStr + '"'
  191. }
  192. personJson = getJson(session, personUrl, asHeader, None,
  193. projectVersion, httpPrefix, domain)
  194. if not personJson:
  195. asHeader = {
  196. 'Accept': 'application/ld+json; profile="' + profileStr + '"'
  197. }
  198. personJson = getJson(session, personUrl, asHeader, None,
  199. projectVersion, httpPrefix, domain)
  200. if not personJson:
  201. print('Unable to get actor')
  202. return None, None, None, None, None, None, None, None
  203. boxJson = None
  204. if not personJson.get(boxName):
  205. if personJson.get('endpoints'):
  206. if personJson['endpoints'].get(boxName):
  207. boxJson = personJson['endpoints'][boxName]
  208. else:
  209. boxJson = personJson[boxName]
  210. if not boxJson:
  211. return None, None, None, None, None, None, None, None
  212. personId = None
  213. if personJson.get('id'):
  214. personId = personJson['id']
  215. pubKeyId = None
  216. pubKey = None
  217. if personJson.get('publicKey'):
  218. if personJson['publicKey'].get('id'):
  219. pubKeyId = personJson['publicKey']['id']
  220. if personJson['publicKey'].get('publicKeyPem'):
  221. pubKey = personJson['publicKey']['publicKeyPem']
  222. sharedInbox = None
  223. if personJson.get('sharedInbox'):
  224. sharedInbox = personJson['sharedInbox']
  225. else:
  226. if personJson.get('endpoints'):
  227. if personJson['endpoints'].get('sharedInbox'):
  228. sharedInbox = personJson['endpoints']['sharedInbox']
  229. capabilityAcquisition = None
  230. if personJson.get('capabilityAcquisitionEndpoint'):
  231. capabilityAcquisition = personJson['capabilityAcquisitionEndpoint']
  232. avatarUrl = None
  233. if personJson.get('icon'):
  234. if personJson['icon'].get('url'):
  235. avatarUrl = personJson['icon']['url']
  236. displayName = None
  237. if personJson.get('name'):
  238. displayName = personJson['name']
  239. storePersonInCache(baseDir, personUrl, personJson, personCache)
  240. return boxJson, pubKeyId, pubKey, personId, sharedInbox, \
  241. capabilityAcquisition, avatarUrl, displayName
  242. def getPosts(session, outboxUrl: str, maxPosts: int,
  243. maxMentions: int,
  244. maxEmoji: int, maxAttachments: int,
  245. federationList: [],
  246. personCache: {}, raw: bool,
  247. simple: bool, debug: bool,
  248. projectVersion: str, httpPrefix: str,
  249. domain: str) -> {}:
  250. """Gets public posts from an outbox
  251. """
  252. personPosts = {}
  253. if not outboxUrl:
  254. return personPosts
  255. profileStr = 'https://www.w3.org/ns/activitystreams'
  256. asHeader = {
  257. 'Accept': 'application/activity+json; profile="' + profileStr + '"'
  258. }
  259. if '/outbox/' in outboxUrl:
  260. asHeader = {
  261. 'Accept': 'application/ld+json; profile="' + profileStr + '"'
  262. }
  263. if raw:
  264. result = []
  265. i = 0
  266. userFeed = parseUserFeed(session, outboxUrl, asHeader,
  267. projectVersion, httpPrefix, domain)
  268. for item in userFeed:
  269. result.append(item)
  270. i += 1
  271. if i == maxPosts:
  272. break
  273. pprint(result)
  274. return None
  275. i = 0
  276. userFeed = parseUserFeed(session, outboxUrl, asHeader,
  277. projectVersion, httpPrefix, domain)
  278. for item in userFeed:
  279. if not item.get('id'):
  280. if debug:
  281. print('No id')
  282. continue
  283. if not item.get('type'):
  284. if debug:
  285. print('No type')
  286. continue
  287. if item['type'] != 'Create':
  288. if debug:
  289. print('Not Create type')
  290. continue
  291. if not item.get('object'):
  292. if debug:
  293. print('No object')
  294. continue
  295. if not isinstance(item['object'], dict):
  296. if debug:
  297. print('item object is not a dict')
  298. continue
  299. if not item['object'].get('published'):
  300. if debug:
  301. print('No published attribute')
  302. continue
  303. if not personPosts.get(item['id']):
  304. # check that this is a public post
  305. # #Public should appear in the "to" list
  306. if item['object'].get('to'):
  307. isPublic = False
  308. for recipient in item['object']['to']:
  309. if recipient.endswith('#Public'):
  310. isPublic = True
  311. break
  312. if not isPublic:
  313. continue
  314. content = \
  315. item['object']['content'].replace('&apos;', "'")
  316. mentions = []
  317. emoji = {}
  318. if item['object'].get('tag'):
  319. for tagItem in item['object']['tag']:
  320. tagType = tagItem['type'].lower()
  321. if tagType == 'emoji':
  322. if tagItem.get('name') and tagItem.get('icon'):
  323. if tagItem['icon'].get('url'):
  324. # No emoji from non-permitted domains
  325. if urlPermitted(tagItem['icon']['url'],
  326. federationList,
  327. "objects:read"):
  328. emojiName = tagItem['name']
  329. emojiIcon = tagItem['icon']['url']
  330. emoji[emojiName] = emojiIcon
  331. else:
  332. if debug:
  333. print('url not permitted ' +
  334. tagItem['icon']['url'])
  335. if tagType == 'mention':
  336. if tagItem.get('name'):
  337. if tagItem['name'] not in mentions:
  338. mentions.append(tagItem['name'])
  339. if len(mentions) > maxMentions:
  340. if debug:
  341. print('max mentions reached')
  342. continue
  343. if len(emoji) > maxEmoji:
  344. if debug:
  345. print('max emojis reached')
  346. continue
  347. summary = ''
  348. if item['object'].get('summary'):
  349. if item['object']['summary']:
  350. summary = item['object']['summary']
  351. inReplyTo = ''
  352. if item['object'].get('inReplyTo'):
  353. if item['object']['inReplyTo']:
  354. # No replies to non-permitted domains
  355. if not urlPermitted(item['object']['inReplyTo'],
  356. federationList,
  357. "objects:read"):
  358. if debug:
  359. print('url not permitted ' +
  360. item['object']['inReplyTo'])
  361. continue
  362. inReplyTo = item['object']['inReplyTo']
  363. conversation = ''
  364. if item['object'].get('conversation'):
  365. if item['object']['conversation']:
  366. # no conversations originated in non-permitted domains
  367. if urlPermitted(item['object']['conversation'],
  368. federationList, "objects:read"):
  369. conversation = item['object']['conversation']
  370. attachment = []
  371. if item['object'].get('attachment'):
  372. if item['object']['attachment']:
  373. for attach in item['object']['attachment']:
  374. if attach.get('name') and attach.get('url'):
  375. # no attachments from non-permitted domains
  376. if urlPermitted(attach['url'],
  377. federationList,
  378. "objects:read"):
  379. attachment.append([attach['name'],
  380. attach['url']])
  381. else:
  382. if debug:
  383. print('url not permitted ' +
  384. attach['url'])
  385. sensitive = False
  386. if item['object'].get('sensitive'):
  387. sensitive = item['object']['sensitive']
  388. if simple:
  389. print(cleanHtml(content) + '\n')
  390. else:
  391. pprint(item)
  392. personPosts[item['id']] = {
  393. "sensitive": sensitive,
  394. "inreplyto": inReplyTo,
  395. "summary": summary,
  396. "html": content,
  397. "plaintext": cleanHtml(content),
  398. "attachment": attachment,
  399. "mentions": mentions,
  400. "emoji": emoji,
  401. "conversation": conversation
  402. }
  403. i += 1
  404. if i == maxPosts:
  405. break
  406. return personPosts
  407. def getPostDomains(session, outboxUrl: str, maxPosts: int,
  408. maxMentions: int,
  409. maxEmoji: int, maxAttachments: int,
  410. federationList: [],
  411. personCache: {},
  412. debug: bool,
  413. projectVersion: str, httpPrefix: str,
  414. domain: str, domainList=[]) -> []:
  415. """Returns a list of domains referenced within public posts
  416. """
  417. if not outboxUrl:
  418. return []
  419. profileStr = 'https://www.w3.org/ns/activitystreams'
  420. asHeader = {
  421. 'Accept': 'application/activity+json; profile="' + profileStr + '"'
  422. }
  423. if '/outbox/' in outboxUrl:
  424. asHeader = {
  425. 'Accept': 'application/ld+json; profile="' + profileStr + '"'
  426. }
  427. postDomains = domainList
  428. i = 0
  429. userFeed = parseUserFeed(session, outboxUrl, asHeader,
  430. projectVersion, httpPrefix, domain)
  431. for item in userFeed:
  432. i += 1
  433. if i > maxPosts:
  434. break
  435. if not item.get('object'):
  436. continue
  437. if not isinstance(item['object'], dict):
  438. continue
  439. if item['object'].get('inReplyTo'):
  440. postDomain, postPort = \
  441. getDomainFromActor(item['object']['inReplyTo'])
  442. if postDomain not in postDomains:
  443. postDomains.append(postDomain)
  444. if item['object'].get('tag'):
  445. for tagItem in item['object']['tag']:
  446. tagType = tagItem['type'].lower()
  447. if tagType == 'mention':
  448. if tagItem.get('href'):
  449. postDomain, postPort = \
  450. getDomainFromActor(tagItem['href'])
  451. if postDomain not in postDomains:
  452. postDomains.append(postDomain)
  453. return postDomains
  454. def deleteAllPosts(baseDir: str,
  455. nickname: str, domain: str, boxname: str) -> None:
  456. """Deletes all posts for a person from inbox or outbox
  457. """
  458. if boxname != 'inbox' and boxname != 'outbox' and boxname != 'tlblogs':
  459. return
  460. boxDir = createPersonDir(nickname, domain, baseDir, boxname)
  461. for deleteFilename in os.scandir(boxDir):
  462. deleteFilename = deleteFilename.name
  463. filePath = os.path.join(boxDir, deleteFilename)
  464. try:
  465. if os.path.isfile(filePath):
  466. os.unlink(filePath)
  467. elif os.path.isdir(filePath):
  468. shutil.rmtree(filePath)
  469. except Exception as e:
  470. print(e)
  471. def savePostToBox(baseDir: str, httpPrefix: str, postId: str,
  472. nickname: str, domain: str, postJsonObject: {},
  473. boxname: str) -> str:
  474. """Saves the give json to the give box
  475. Returns the filename
  476. """
  477. if boxname != 'inbox' and boxname != 'outbox' and \
  478. boxname != 'tlblogs' and boxname != 'scheduled':
  479. return None
  480. originalDomain = domain
  481. if ':' in domain:
  482. domain = domain.split(':')[0]
  483. if not postId:
  484. statusNumber, published = getStatusNumber()
  485. postId = \
  486. httpPrefix + '://' + originalDomain + '/users/' + nickname + \
  487. '/statuses/' + statusNumber
  488. postJsonObject['id'] = postId + '/activity'
  489. if postJsonObject.get('object'):
  490. if isinstance(postJsonObject['object'], dict):
  491. postJsonObject['object']['id'] = postId
  492. postJsonObject['object']['atomUri'] = postId
  493. boxDir = createPersonDir(nickname, domain, baseDir, boxname)
  494. filename = boxDir + '/' + postId.replace('/', '#') + '.json'
  495. saveJson(postJsonObject, filename)
  496. return filename
  497. def updateHashtagsIndex(baseDir: str, tag: {}, newPostId: str) -> None:
  498. """Writes the post url for hashtags to a file
  499. This allows posts for a hashtag to be quickly looked up
  500. """
  501. if tag['type'] != 'Hashtag':
  502. return
  503. # create hashtags directory
  504. tagsDir = baseDir + '/tags'
  505. if not os.path.isdir(tagsDir):
  506. os.mkdir(tagsDir)
  507. tagName = tag['name']
  508. tagsFilename = tagsDir + '/' + tagName[1:] + '.txt'
  509. tagline = newPostId + '\n'
  510. if not os.path.isfile(tagsFilename):
  511. # create a new tags index file
  512. tagsFile = open(tagsFilename, "w+")
  513. if tagsFile:
  514. tagsFile.write(tagline)
  515. tagsFile.close()
  516. else:
  517. # prepend to tags index file
  518. if tagline not in open(tagsFilename).read():
  519. try:
  520. with open(tagsFilename, 'r+') as tagsFile:
  521. content = tagsFile.read()
  522. tagsFile.seek(0, 0)
  523. tagsFile.write(tagline+content)
  524. except Exception as e:
  525. print('WARN: Failed to write entry to tags file ' +
  526. tagsFilename + ' ' + str(e))
  527. def addSchedulePost(baseDir: str, nickname: str, domain: str,
  528. eventDateStr: str, postId: str) -> None:
  529. """Adds a scheduled post to the index
  530. """
  531. handle = nickname + '@' + domain
  532. scheduleIndexFilename = baseDir + '/accounts/' + handle + '/schedule.index'
  533. indexStr = eventDateStr + ' ' + postId.replace('/', '#')
  534. if os.path.isfile(scheduleIndexFilename):
  535. if indexStr not in open(scheduleIndexFilename).read():
  536. try:
  537. with open(scheduleIndexFilename, 'r+') as scheduleFile:
  538. content = scheduleFile.read()
  539. scheduleFile.seek(0, 0)
  540. scheduleFile.write(indexStr + '\n' + content)
  541. print('DEBUG: scheduled post added to index')
  542. except Exception as e:
  543. print('WARN: Failed to write entry to scheduled posts index ' +
  544. scheduleIndexFilename + ' ' + str(e))
  545. else:
  546. scheduleFile = open(scheduleIndexFilename, 'w')
  547. if scheduleFile:
  548. scheduleFile.write(indexStr + '\n')
  549. scheduleFile.close()
  550. def createPostBase(baseDir: str, nickname: str, domain: str, port: int,
  551. toUrl: str, ccUrl: str, httpPrefix: str, content: str,
  552. followersOnly: bool, saveToFile: bool, clientToServer: bool,
  553. attachImageFilename: str,
  554. mediaType: str, imageDescription: str,
  555. useBlurhash: bool, isModerationReport: bool,
  556. isArticle: bool, inReplyTo=None,
  557. inReplyToAtomUri=None, subject=None, schedulePost=False,
  558. eventDate=None, eventTime=None, location=None) -> {}:
  559. """Creates a message
  560. """
  561. mentionedRecipients = \
  562. getMentionedPeople(baseDir, httpPrefix, content, domain, False)
  563. tags = []
  564. hashtagsDict = {}
  565. if port:
  566. if port != 80 and port != 443:
  567. if ':' not in domain:
  568. domain = domain + ':' + str(port)
  569. # add tags
  570. content = \
  571. addHtmlTags(baseDir, httpPrefix,
  572. nickname, domain, content,
  573. mentionedRecipients,
  574. hashtagsDict, True)
  575. # replace emoji with unicode
  576. tags = []
  577. for tagName, tag in hashtagsDict.items():
  578. tags.append(tag)
  579. # get list of tags
  580. content = replaceEmojiFromTags(content, tags, 'content')
  581. # remove replaced emoji
  582. hashtagsDictCopy = hashtagsDict.copy()
  583. for tagName, tag in hashtagsDictCopy.items():
  584. if tag.get('name'):
  585. if tag['name'].startswith(':'):
  586. if tag['name'] not in content:
  587. del hashtagsDict[tagName]
  588. statusNumber, published = getStatusNumber()
  589. newPostId = \
  590. httpPrefix + '://' + domain + '/users/' + \
  591. nickname + '/statuses/' + statusNumber
  592. sensitive = False
  593. summary = None
  594. if subject:
  595. summary = subject
  596. sensitive = True
  597. toRecipients = []
  598. toCC = []
  599. if toUrl:
  600. if not isinstance(toUrl, str):
  601. print('ERROR: toUrl is not a string')
  602. return None
  603. toRecipients = [toUrl]
  604. # who to send to
  605. if mentionedRecipients:
  606. for mention in mentionedRecipients:
  607. if mention not in toCC:
  608. toCC.append(mention)
  609. # create a list of hashtags
  610. # Only posts which are #Public are searchable by hashtag
  611. if hashtagsDict:
  612. isPublic = False
  613. for recipient in toRecipients:
  614. if recipient.endswith('#Public'):
  615. isPublic = True
  616. break
  617. for tagName, tag in hashtagsDict.items():
  618. tags.append(tag)
  619. if isPublic:
  620. updateHashtagsIndex(baseDir, tag, newPostId)
  621. print('Content tags: ' + str(tags))
  622. if inReplyTo and not sensitive:
  623. # locate the post which this is a reply to and check if
  624. # it has a content warning. If it does then reproduce
  625. # the same warning
  626. replyPostFilename = \
  627. locatePost(baseDir, nickname, domain, inReplyTo)
  628. if replyPostFilename:
  629. replyToJson = loadJson(replyPostFilename)
  630. if replyToJson:
  631. if replyToJson.get('object'):
  632. if replyToJson['object'].get('sensitive'):
  633. if replyToJson['object']['sensitive']:
  634. sensitive = True
  635. if replyToJson['object'].get('summary'):
  636. summary = replyToJson['object']['summary']
  637. eventDateStr = None
  638. if eventDate:
  639. eventName = summary
  640. if not eventName:
  641. eventName = content
  642. eventDateStr = eventDate
  643. if eventTime:
  644. if eventTime.endswith('Z'):
  645. eventDateStr = eventDate + 'T' + eventTime
  646. else:
  647. eventDateStr = eventDate + 'T' + eventTime + \
  648. ':00' + strftime("%z", gmtime())
  649. else:
  650. eventDateStr = eventDate + 'T12:00:00Z'
  651. if not schedulePost:
  652. tags.append({
  653. "@context": "https://www.w3.org/ns/activitystreams",
  654. "type": "Event",
  655. "name": eventName,
  656. "startTime": eventDateStr,
  657. "endTime": eventDateStr
  658. })
  659. if location:
  660. tags.append({
  661. "@context": "https://www.w3.org/ns/activitystreams",
  662. "type": "Place",
  663. "name": location
  664. })
  665. postContext = [
  666. 'https://www.w3.org/ns/activitystreams',
  667. {
  668. 'Hashtag': 'as:Hashtag',
  669. 'sensitive': 'as:sensitive',
  670. 'toot': 'http://joinmastodon.org/ns#',
  671. 'votersCount': 'toot:votersCount'
  672. }
  673. ]
  674. # make sure that CC doesn't also contain a To address
  675. # eg. To: [ "https://mydomain/users/foo/followers" ]
  676. # CC: [ "X", "Y", "https://mydomain/users/foo", "Z" ]
  677. removeFromCC = []
  678. for ccRecipient in toCC:
  679. for sendToActor in toRecipients:
  680. if ccRecipient in sendToActor and \
  681. ccRecipient not in removeFromCC:
  682. removeFromCC.append(ccRecipient)
  683. break
  684. for ccRemoval in removeFromCC:
  685. toCC.remove(ccRemoval)
  686. if not clientToServer:
  687. actorUrl = httpPrefix + '://' + domain + '/users/' + nickname
  688. # if capabilities have been granted for this actor
  689. # then get the corresponding id
  690. capabilityIdList = []
  691. ocapFilename = getOcapFilename(baseDir, nickname, domain,
  692. toUrl, 'granted')
  693. if ocapFilename:
  694. if os.path.isfile(ocapFilename):
  695. oc = loadJson(ocapFilename)
  696. if oc:
  697. if oc.get('id'):
  698. capabilityIdList = [oc['id']]
  699. idStr = \
  700. httpPrefix + '://' + domain + '/users/' + nickname + \
  701. '/statuses/' + statusNumber + '/replies'
  702. newPost = {
  703. '@context': postContext,
  704. 'id': newPostId+'/activity',
  705. 'capability': capabilityIdList,
  706. 'type': 'Create',
  707. 'actor': actorUrl,
  708. 'published': published,
  709. 'to': toRecipients,
  710. 'cc': toCC,
  711. 'object': {
  712. 'id': newPostId,
  713. 'type': 'Note',
  714. 'summary': summary,
  715. 'inReplyTo': inReplyTo,
  716. 'published': published,
  717. 'url': httpPrefix+'://'+domain+'/@'+nickname+'/'+statusNumber,
  718. 'attributedTo': httpPrefix+'://'+domain+'/users/'+nickname,
  719. 'to': toRecipients,
  720. 'cc': toCC,
  721. 'sensitive': sensitive,
  722. 'atomUri': newPostId,
  723. 'inReplyToAtomUri': inReplyToAtomUri,
  724. 'mediaType': 'text/html',
  725. 'content': content,
  726. 'contentMap': {
  727. 'en': content
  728. },
  729. 'attachment': [],
  730. 'tag': tags,
  731. 'replies': {
  732. 'id': idStr,
  733. 'type': 'Collection',
  734. 'first': {
  735. 'type': 'CollectionPage',
  736. 'partOf': idStr,
  737. 'items': []
  738. }
  739. }
  740. }
  741. }
  742. if attachImageFilename:
  743. newPost['object'] = \
  744. attachMedia(baseDir, httpPrefix, domain, port,
  745. newPost['object'], attachImageFilename,
  746. mediaType, imageDescription, useBlurhash)
  747. else:
  748. idStr = \
  749. httpPrefix + '://' + domain + '/users/' + nickname + \
  750. '/statuses/' + statusNumber + '/replies'
  751. newPost = {
  752. "@context": postContext,
  753. 'id': newPostId,
  754. 'type': 'Note',
  755. 'summary': summary,
  756. 'inReplyTo': inReplyTo,
  757. 'published': published,
  758. 'url': httpPrefix+'://'+domain+'/@'+nickname+'/'+statusNumber,
  759. 'attributedTo': httpPrefix+'://'+domain+'/users/'+nickname,
  760. 'to': toRecipients,
  761. 'cc': toCC,
  762. 'sensitive': sensitive,
  763. 'atomUri': newPostId,
  764. 'inReplyToAtomUri': inReplyToAtomUri,
  765. 'mediaType': 'text/html',
  766. 'content': content,
  767. 'contentMap': {
  768. 'en': content
  769. },
  770. 'attachment': [],
  771. 'tag': tags,
  772. 'replies': {
  773. 'id': idStr,
  774. 'type': 'Collection',
  775. 'first': {
  776. 'type': 'CollectionPage',
  777. 'partOf': idStr,
  778. 'items': []
  779. }
  780. }
  781. }
  782. if attachImageFilename:
  783. newPost = \
  784. attachMedia(baseDir, httpPrefix, domain, port,
  785. newPost, attachImageFilename,
  786. mediaType, imageDescription, useBlurhash)
  787. if ccUrl:
  788. if len(ccUrl) > 0:
  789. newPost['cc'] = [ccUrl]
  790. if newPost.get('object'):
  791. newPost['object']['cc'] = [ccUrl]
  792. # if this is a moderation report then add a status
  793. if isModerationReport:
  794. # add status
  795. if newPost.get('object'):
  796. newPost['object']['moderationStatus'] = 'pending'
  797. else:
  798. newPost['moderationStatus'] = 'pending'
  799. # save to index file
  800. moderationIndexFile = baseDir + '/accounts/moderation.txt'
  801. modFile = open(moderationIndexFile, "a+")
  802. if modFile:
  803. modFile.write(newPostId + '\n')
  804. modFile.close()
  805. # If a patch has been posted - i.e. the output from
  806. # git format-patch - then convert the activitypub type
  807. convertPostToPatch(baseDir, nickname, domain, newPost)
  808. if schedulePost:
  809. if eventDate and eventTime:
  810. # add an item to the scheduled post index file
  811. addSchedulePost(baseDir, nickname, domain, eventDateStr, newPostId)
  812. savePostToBox(baseDir, httpPrefix, newPostId,
  813. nickname, domain, newPost, 'scheduled')
  814. else:
  815. print('Unable to create scheduled post without ' +
  816. 'date and time values')
  817. return newPost
  818. elif saveToFile:
  819. if not isArticle:
  820. savePostToBox(baseDir, httpPrefix, newPostId,
  821. nickname, domain, newPost, 'outbox')
  822. else:
  823. savePostToBox(baseDir, httpPrefix, newPostId,
  824. nickname, domain, newPost, 'tlblogs')
  825. return newPost
  826. def outboxMessageCreateWrap(httpPrefix: str,
  827. nickname: str, domain: str, port: int,
  828. messageJson: {}) -> {}:
  829. """Wraps a received message in a Create
  830. https://www.w3.org/TR/activitypub/#object-without-create
  831. """
  832. if port:
  833. if port != 80 and port != 443:
  834. if ':' not in domain:
  835. domain = domain + ':' + str(port)
  836. statusNumber, published = getStatusNumber()
  837. if messageJson.get('published'):
  838. published = messageJson['published']
  839. newPostId = \
  840. httpPrefix + '://' + domain + '/users/' + nickname + \
  841. '/statuses/' + statusNumber
  842. cc = []
  843. if messageJson.get('cc'):
  844. cc = messageJson['cc']
  845. capabilityUrl = []
  846. newPost = {
  847. "@context": "https://www.w3.org/ns/activitystreams",
  848. 'id': newPostId+'/activity',
  849. 'capability': capabilityUrl,
  850. 'type': 'Create',
  851. 'actor': httpPrefix+'://'+domain+'/users/'+nickname,
  852. 'published': published,
  853. 'to': messageJson['to'],
  854. 'cc': cc,
  855. 'object': messageJson
  856. }
  857. newPost['object']['id'] = newPost['id']
  858. newPost['object']['url'] = \
  859. httpPrefix + '://' + domain + '/@' + nickname + '/' + statusNumber
  860. newPost['object']['atomUri'] = \
  861. httpPrefix + '://' + domain + '/users/' + nickname + \
  862. '/statuses/' + statusNumber
  863. return newPost
  864. def postIsAddressedToFollowers(baseDir: str,
  865. nickname: str, domain: str, port: int,
  866. httpPrefix: str,
  867. postJsonObject: {}) -> bool:
  868. """Returns true if the given post is addressed to followers of the nickname
  869. """
  870. if port:
  871. if port != 80 and port != 443:
  872. if ':' not in domain:
  873. domain = domain + ':' + str(port)
  874. if not postJsonObject.get('object'):
  875. return False
  876. toList = []
  877. ccList = []
  878. if postJsonObject['type'] != 'Update' and \
  879. isinstance(postJsonObject['object'], dict):
  880. if postJsonObject['object'].get('to'):
  881. toList = postJsonObject['object']['to']
  882. if postJsonObject['object'].get('cc'):
  883. ccList = postJsonObject['object']['cc']
  884. else:
  885. if postJsonObject.get('to'):
  886. toList = postJsonObject['to']
  887. if postJsonObject.get('cc'):
  888. ccList = postJsonObject['cc']
  889. followersUrl = httpPrefix + '://' + domain + '/users/' + \
  890. nickname + '/followers'
  891. # does the followers url exist in 'to' or 'cc' lists?
  892. addressedToFollowers = False
  893. if followersUrl in toList:
  894. addressedToFollowers = True
  895. elif followersUrl in ccList:
  896. addressedToFollowers = True
  897. return addressedToFollowers
  898. def postIsAddressedToPublic(baseDir: str, postJsonObject: {}) -> bool:
  899. """Returns true if the given post is addressed to public
  900. """
  901. if not postJsonObject.get('object'):
  902. return False
  903. if not postJsonObject['object'].get('to'):
  904. return False
  905. publicUrl = 'https://www.w3.org/ns/activitystreams#Public'
  906. # does the public url exist in 'to' or 'cc' lists?
  907. addressedToPublic = False
  908. if publicUrl in postJsonObject['object']['to']:
  909. addressedToPublic = True
  910. if not addressedToPublic:
  911. if not postJsonObject['object'].get('cc'):
  912. return False
  913. if publicUrl in postJsonObject['object']['cc']:
  914. addressedToPublic = True
  915. return addressedToPublic
  916. def createPublicPost(baseDir: str,
  917. nickname: str, domain: str, port: int, httpPrefix: str,
  918. content: str, followersOnly: bool, saveToFile: bool,
  919. clientToServer: bool,
  920. attachImageFilename: str, mediaType: str,
  921. imageDescription: str, useBlurhash: bool,
  922. inReplyTo=None, inReplyToAtomUri=None, subject=None,
  923. schedulePost=False,
  924. eventDate=None, eventTime=None, location=None) -> {}:
  925. """Public post
  926. """
  927. domainFull = domain
  928. if port:
  929. if port != 80 and port != 443:
  930. if ':' not in domain:
  931. domainFull = domain + ':' + str(port)
  932. return createPostBase(baseDir, nickname, domain, port,
  933. 'https://www.w3.org/ns/activitystreams#Public',
  934. httpPrefix + '://' + domainFull + '/users/' +
  935. nickname + '/followers',
  936. httpPrefix, content, followersOnly, saveToFile,
  937. clientToServer,
  938. attachImageFilename, mediaType,
  939. imageDescription, useBlurhash,
  940. False, False, inReplyTo, inReplyToAtomUri, subject,
  941. schedulePost, eventDate, eventTime, location)
  942. def createBlogPost(baseDir: str,
  943. nickname: str, domain: str, port: int, httpPrefix: str,
  944. content: str, followersOnly: bool, saveToFile: bool,
  945. clientToServer: bool,
  946. attachImageFilename: str, mediaType: str,
  947. imageDescription: str, useBlurhash: bool,
  948. inReplyTo=None, inReplyToAtomUri=None, subject=None,
  949. schedulePost=False,
  950. eventDate=None, eventTime=None, location=None) -> {}:
  951. blog = \
  952. createPublicPost(baseDir,
  953. nickname, domain, port, httpPrefix,
  954. content, followersOnly, saveToFile,
  955. clientToServer,
  956. attachImageFilename, mediaType,
  957. imageDescription, useBlurhash,
  958. inReplyTo, inReplyToAtomUri, subject,
  959. schedulePost,
  960. eventDate, eventTime, location)
  961. blog['object']['type'] = 'Article'
  962. return blog
  963. def createQuestionPost(baseDir: str,
  964. nickname: str, domain: str, port: int, httpPrefix: str,
  965. content: str, qOptions: [],
  966. followersOnly: bool, saveToFile: bool,
  967. clientToServer: bool,
  968. attachImageFilename: str, mediaType: str,
  969. imageDescription: str, useBlurhash: bool,
  970. subject: str, durationDays: int) -> {}:
  971. """Question post with multiple choice options
  972. """
  973. domainFull = domain
  974. if port:
  975. if port != 80 and port != 443:
  976. if ':' not in domain:
  977. domainFull = domain + ':' + str(port)
  978. messageJson = \
  979. createPostBase(baseDir, nickname, domain, port,
  980. 'https://www.w3.org/ns/activitystreams#Public',
  981. httpPrefix + '://' + domainFull + '/users/' +
  982. nickname + '/followers',
  983. httpPrefix, content, followersOnly, saveToFile,
  984. clientToServer,
  985. attachImageFilename, mediaType,
  986. imageDescription, useBlurhash,
  987. False, False, None, None, subject,
  988. False, None, None, None)
  989. messageJson['object']['type'] = 'Question'
  990. messageJson['object']['oneOf'] = []
  991. messageJson['object']['votersCount'] = 0
  992. currTime = datetime.datetime.utcnow()
  993. daysSinceEpoch = \
  994. int((currTime - datetime.datetime(1970, 1, 1)).days + durationDays)
  995. endTime = datetime.datetime(1970, 1, 1) + \
  996. datetime.timedelta(daysSinceEpoch)
  997. messageJson['object']['endTime'] = endTime.strftime("%Y-%m-%dT%H:%M:%SZ")
  998. for questionOption in qOptions:
  999. messageJson['object']['oneOf'].append({
  1000. "type": "Note",
  1001. "name": questionOption,
  1002. "replies": {
  1003. "type": "Collection",
  1004. "totalItems": 0
  1005. }
  1006. })
  1007. return messageJson
  1008. def createUnlistedPost(baseDir: str,
  1009. nickname: str, domain: str, port: int, httpPrefix: str,
  1010. content: str, followersOnly: bool, saveToFile: bool,
  1011. clientToServer: bool,
  1012. attachImageFilename: str, mediaType: str,
  1013. imageDescription: str, useBlurhash: bool,
  1014. inReplyTo=None, inReplyToAtomUri=None, subject=None,
  1015. schedulePost=False,
  1016. eventDate=None, eventTime=None, location=None) -> {}:
  1017. """Unlisted post. This has the #Public and followers links inverted.
  1018. """
  1019. domainFull = domain
  1020. if port:
  1021. if port != 80 and port != 443:
  1022. if ':' not in domain:
  1023. domainFull = domain + ':' + str(port)
  1024. return createPostBase(baseDir, nickname, domain, port,
  1025. httpPrefix + '://' + domainFull + '/users/' +
  1026. nickname + '/followers',
  1027. 'https://www.w3.org/ns/activitystreams#Public',
  1028. httpPrefix, content, followersOnly, saveToFile,
  1029. clientToServer,
  1030. attachImageFilename, mediaType,
  1031. imageDescription, useBlurhash,
  1032. False, False, inReplyTo, inReplyToAtomUri, subject,
  1033. schedulePost, eventDate, eventTime, location)
  1034. def createFollowersOnlyPost(baseDir: str,
  1035. nickname: str, domain: str, port: int,
  1036. httpPrefix: str,
  1037. content: str, followersOnly: bool,
  1038. saveToFile: bool,
  1039. clientToServer: bool,
  1040. attachImageFilename: str, mediaType: str,
  1041. imageDescription: str, useBlurhash: bool,
  1042. inReplyTo=None, inReplyToAtomUri=None,
  1043. subject=None, schedulePost=False,
  1044. eventDate=None, eventTime=None,
  1045. location=None) -> {}:
  1046. """Followers only post
  1047. """
  1048. domainFull = domain
  1049. if port:
  1050. if port != 80 and port != 443:
  1051. if ':' not in domain:
  1052. domainFull = domain + ':' + str(port)
  1053. return createPostBase(baseDir, nickname, domain, port,
  1054. httpPrefix + '://' + domainFull + '/users/' +
  1055. nickname + '/followers',
  1056. None,
  1057. httpPrefix, content, followersOnly, saveToFile,
  1058. clientToServer,
  1059. attachImageFilename, mediaType,
  1060. imageDescription, useBlurhash,
  1061. False, False, inReplyTo, inReplyToAtomUri, subject,
  1062. schedulePost, eventDate, eventTime, location)
  1063. def getMentionedPeople(baseDir: str, httpPrefix: str,
  1064. content: str, domain: str, debug: bool) -> []:
  1065. """Extracts a list of mentioned actors from the given message content
  1066. """
  1067. if '@' not in content:
  1068. return None
  1069. mentions = []
  1070. words = content.split(' ')
  1071. for wrd in words:
  1072. if wrd.startswith('@'):
  1073. handle = wrd[1:]
  1074. if debug:
  1075. print('DEBUG: mentioned handle ' + handle)
  1076. if '@' not in handle:
  1077. handle = handle + '@' + domain
  1078. if not os.path.isdir(baseDir + '/accounts/' + handle):
  1079. continue
  1080. else:
  1081. externalDomain = handle.split('@')[1]
  1082. if not ('.' in externalDomain or
  1083. externalDomain == 'localhost'):
  1084. continue
  1085. mentionedNickname = handle.split('@')[0]
  1086. mentionedDomain = handle.split('@')[1].strip('\n').strip('\r')
  1087. if ':' in mentionedDomain:
  1088. mentionedDomain = mentionedDomain.split(':')[0]
  1089. if not validNickname(mentionedDomain, mentionedNickname):
  1090. continue
  1091. actor = \
  1092. httpPrefix + '://' + handle.split('@')[1] + \
  1093. '/users/' + mentionedNickname
  1094. mentions.append(actor)
  1095. return mentions
  1096. def createDirectMessagePost(baseDir: str,
  1097. nickname: str, domain: str, port: int,
  1098. httpPrefix: str,
  1099. content: str, followersOnly: bool,
  1100. saveToFile: bool, clientToServer: bool,
  1101. attachImageFilename: str, mediaType: str,
  1102. imageDescription: str, useBlurhash: bool,
  1103. inReplyTo=None, inReplyToAtomUri=None,
  1104. subject=None, debug=False,
  1105. schedulePost=False,
  1106. eventDate=None, eventTime=None,
  1107. location=None) -> {}:
  1108. """Direct Message post
  1109. """
  1110. content = resolvePetnames(baseDir, nickname, domain, content)
  1111. mentionedPeople = \
  1112. getMentionedPeople(baseDir, httpPrefix, content, domain, debug)
  1113. if debug:
  1114. print('mentionedPeople: ' + str(mentionedPeople))
  1115. if not mentionedPeople:
  1116. return None
  1117. postTo = None
  1118. postCc = None
  1119. messageJson = \
  1120. createPostBase(baseDir, nickname, domain, port,
  1121. postTo, postCc,
  1122. httpPrefix, content, followersOnly, saveToFile,
  1123. clientToServer,
  1124. attachImageFilename, mediaType,
  1125. imageDescription, useBlurhash,
  1126. False, False, inReplyTo, inReplyToAtomUri, subject,
  1127. schedulePost, eventDate, eventTime, location)
  1128. # mentioned recipients go into To rather than Cc
  1129. messageJson['to'] = messageJson['object']['cc']
  1130. messageJson['object']['to'] = messageJson['to']
  1131. messageJson['cc'] = []
  1132. messageJson['object']['cc'] = []
  1133. if schedulePost:
  1134. savePostToBox(baseDir, httpPrefix, messageJson['object']['id'],
  1135. nickname, domain, messageJson, 'scheduled')
  1136. return messageJson
  1137. def createReportPost(baseDir: str,
  1138. nickname: str, domain: str, port: int, httpPrefix: str,
  1139. content: str, followersOnly: bool, saveToFile: bool,
  1140. clientToServer: bool,
  1141. attachImageFilename: str, mediaType: str,
  1142. imageDescription: str, useBlurhash: bool,
  1143. debug: bool, subject=None) -> {}:
  1144. """Send a report to moderators
  1145. """
  1146. domainFull = domain
  1147. if port:
  1148. if port != 80 and port != 443:
  1149. if ':' not in domain:
  1150. domainFull = domain + ':' + str(port)
  1151. # add a title to distinguish moderation reports from other posts
  1152. reportTitle = 'Moderation Report'
  1153. if not subject:
  1154. subject = reportTitle
  1155. else:
  1156. if not subject.startswith(reportTitle):
  1157. subject = reportTitle + ': ' + subject
  1158. # create the list of moderators from the moderators file
  1159. moderatorsList = []
  1160. moderatorsFile = baseDir + '/accounts/moderators.txt'
  1161. if os.path.isfile(moderatorsFile):
  1162. with open(moderatorsFile, "r") as fileHandler:
  1163. for line in fileHandler:
  1164. line = line.strip('\n').strip('\r')
  1165. if line.startswith('#'):
  1166. continue
  1167. if line.startswith('/users/'):
  1168. line = line.replace('users', '')
  1169. if line.startswith('@'):
  1170. line = line[1:]
  1171. if '@' in line:
  1172. moderatorActor = httpPrefix + '://' + domainFull + \
  1173. '/users/' + line.split('@')[0]
  1174. if moderatorActor not in moderatorsList:
  1175. moderatorsList.append(moderatorActor)
  1176. continue
  1177. if line.startswith('http') or line.startswith('dat'):
  1178. # must be a local address - no remote moderators
  1179. if '://' + domainFull + '/' in line:
  1180. if line not in moderatorsList:
  1181. moderatorsList.append(line)
  1182. else:
  1183. if '/' not in line:
  1184. moderatorActor = httpPrefix + '://' + domainFull + \
  1185. '/users/' + line
  1186. if moderatorActor not in moderatorsList:
  1187. moderatorsList.append(moderatorActor)
  1188. if len(moderatorsList) == 0:
  1189. # if there are no moderators then the admin becomes the moderator
  1190. adminNickname = getConfigParam(baseDir, 'admin')
  1191. if adminNickname:
  1192. moderatorsList.append(httpPrefix + '://' + domainFull +
  1193. '/users/' + adminNickname)
  1194. if not moderatorsList:
  1195. return None
  1196. if debug:
  1197. print('DEBUG: Sending report to moderators')
  1198. print(str(moderatorsList))
  1199. postTo = moderatorsList
  1200. postCc = None
  1201. postJsonObject = None
  1202. for toUrl in postTo:
  1203. # who is this report going to?
  1204. toNickname = toUrl.split('/users/')[1]
  1205. handle = toNickname + '@' + domain
  1206. postJsonObject = \
  1207. createPostBase(baseDir, nickname, domain, port,
  1208. toUrl, postCc,
  1209. httpPrefix, content, followersOnly, saveToFile,
  1210. clientToServer,
  1211. attachImageFilename, mediaType,
  1212. imageDescription, useBlurhash,
  1213. True, False, None, None, subject,
  1214. False, None, None, None)
  1215. if not postJsonObject:
  1216. continue
  1217. # update the inbox index with the report filename
  1218. # indexFilename=baseDir+'/accounts/'+handle+'/inbox.index'
  1219. # indexEntry=postJsonObject['id'].replace('/activity','').replace('/','#')+'.json'
  1220. # if indexEntry not in open(indexFilename).read():
  1221. # try:
  1222. # with open(indexFilename, 'a+') as fp:
  1223. # fp.write(indexEntry)
  1224. # except:
  1225. # pass
  1226. # save a notification file so that the moderator
  1227. # knows something new has appeared
  1228. newReportFile = baseDir + '/accounts/' + handle + '/.newReport'
  1229. if os.path.isfile(newReportFile):
  1230. continue
  1231. try:
  1232. with open(newReportFile, 'w') as fp:
  1233. fp.write(toUrl + '/moderation')
  1234. except BaseException:
  1235. pass
  1236. return postJsonObject
  1237. def threadSendPost(session, postJsonStr: str, federationList: [],
  1238. inboxUrl: str, baseDir: str,
  1239. signatureHeaderJson: {}, postLog: [],
  1240. debug: bool) -> None:
  1241. """Sends a with retries
  1242. """
  1243. tries = 0
  1244. sendIntervalSec = 30
  1245. for attempt in range(20):
  1246. postResult = None
  1247. unauthorized = False
  1248. try:
  1249. postResult, unauthorized = \
  1250. postJsonString(session, postJsonStr, federationList,
  1251. inboxUrl, signatureHeaderJson,
  1252. "inbox:write", debug)
  1253. except Exception as e:
  1254. print('ERROR: postJsonString failed ' + str(e))
  1255. if unauthorized:
  1256. print(postJsonStr)
  1257. print('threadSendPost: Post is unauthorized')
  1258. break
  1259. if postResult:
  1260. logStr = 'Success on try ' + str(tries) + ': ' + postJsonStr
  1261. else:
  1262. logStr = 'Retry ' + str(tries) + ': ' + postJsonStr
  1263. postLog.append(logStr)
  1264. # keep the length of the log finite
  1265. # Don't accumulate massive files on systems with limited resources
  1266. while len(postLog) > 16:
  1267. postLog.pop(0)
  1268. if debug:
  1269. # save the log file
  1270. postLogFilename = baseDir + '/post.log'
  1271. with open(postLogFilename, "a+") as logFile:
  1272. logFile.write(logStr + '\n')
  1273. if postResult:
  1274. if debug:
  1275. print('DEBUG: successful json post to ' + inboxUrl)
  1276. # our work here is done
  1277. break
  1278. if debug:
  1279. print(postJsonStr)
  1280. print('DEBUG: json post to ' + inboxUrl +
  1281. ' failed. Waiting for ' +
  1282. str(sendIntervalSec) + ' seconds.')
  1283. time.sleep(sendIntervalSec)
  1284. tries += 1
  1285. def sendPost(projectVersion: str,
  1286. session, baseDir: str, nickname: str, domain: str, port: int,
  1287. toNickname: str, toDomain: str, toPort: int, cc: str,
  1288. httpPrefix: str, content: str, followersOnly: bool,
  1289. saveToFile: bool, clientToServer: bool,
  1290. attachImageFilename: str, mediaType: str,
  1291. imageDescription: str, useBlurhash: bool,
  1292. federationList: [], sendThreads: [], postLog: [],
  1293. cachedWebfingers: {}, personCache: {},
  1294. isArticle: bool,
  1295. debug=False, inReplyTo=None,
  1296. inReplyToAtomUri=None, subject=None) -> int:
  1297. """Post to another inbox
  1298. """
  1299. withDigest = True
  1300. if toNickname == 'inbox':
  1301. # shared inbox actor on @domain@domain
  1302. toNickname = toDomain
  1303. if toPort:
  1304. if toPort != 80 and toPort != 443:
  1305. if ':' not in toDomain:
  1306. toDomain = toDomain + ':' + str(toPort)
  1307. handle = httpPrefix + '://' + toDomain + '/@' + toNickname
  1308. # lookup the inbox for the To handle
  1309. wfRequest = webfingerHandle(session, handle, httpPrefix,
  1310. cachedWebfingers,
  1311. domain, projectVersion)
  1312. if not wfRequest:
  1313. return 1
  1314. if not isinstance(wfRequest, dict):
  1315. print('WARN: Webfinger for ' + handle + ' did not return a dict. ' +
  1316. str(wfRequest))
  1317. return 1
  1318. if not clientToServer:
  1319. postToBox = 'inbox'
  1320. else:
  1321. postToBox = 'outbox'
  1322. if isArticle:
  1323. postToBox = 'tlblogs'
  1324. # get the actor inbox for the To handle
  1325. (inboxUrl, pubKeyId, pubKey,
  1326. toPersonId, sharedInbox,
  1327. capabilityAcquisition,
  1328. avatarUrl, displayName) = getPersonBox(baseDir, session, wfRequest,
  1329. personCache,
  1330. projectVersion, httpPrefix,
  1331. nickname, domain, postToBox)
  1332. # If there are more than one followers on the target domain
  1333. # then send to the shared inbox indead of the individual inbox
  1334. if nickname == 'capabilities':
  1335. inboxUrl = capabilityAcquisition
  1336. if not capabilityAcquisition:
  1337. return 2
  1338. if not inboxUrl:
  1339. return 3
  1340. if not pubKey:
  1341. return 4
  1342. if not toPersonId:
  1343. return 5
  1344. # sharedInbox and capabilities are optional
  1345. postJsonObject = \
  1346. createPostBase(baseDir, nickname, domain, port,
  1347. toPersonId, cc, httpPrefix, content,
  1348. followersOnly, saveToFile, clientToServer,
  1349. attachImageFilename, mediaType,
  1350. imageDescription, useBlurhash,
  1351. False, isArticle, inReplyTo,
  1352. inReplyToAtomUri, subject,
  1353. False, None, None, None)
  1354. # get the senders private key
  1355. privateKeyPem = getPersonKey(nickname, domain, baseDir, 'private')
  1356. if len(privateKeyPem) == 0:
  1357. return 6
  1358. if toDomain not in inboxUrl:
  1359. return 7
  1360. postPath = inboxUrl.split(toDomain, 1)[1]
  1361. if not postJsonObject.get('signature'):
  1362. try:
  1363. signedPostJsonObject = jsonldSign(postJsonObject, privateKeyPem)
  1364. postJsonObject = signedPostJsonObject
  1365. except BaseException:
  1366. print('WARN: failed to JSON-LD sign post')
  1367. pass
  1368. # convert json to string so that there are no
  1369. # subsequent conversions after creating message body digest
  1370. postJsonStr = json.dumps(postJsonObject)
  1371. # construct the http header, including the message body digest
  1372. signatureHeaderJson = \
  1373. createSignedHeader(privateKeyPem, nickname, domain, port,
  1374. toDomain, toPort,
  1375. postPath, httpPrefix, withDigest, postJsonStr)
  1376. # Keep the number of threads being used small
  1377. while len(sendThreads) > 1000:
  1378. print('WARN: Maximum threads reached - killing send thread')
  1379. sendThreads[0].kill()
  1380. sendThreads.pop(0)
  1381. print('WARN: thread killed')
  1382. thr = \
  1383. threadWithTrace(target=threadSendPost,
  1384. args=(session,
  1385. postJsonStr,
  1386. federationList,
  1387. inboxUrl, baseDir,
  1388. signatureHeaderJson.copy(),
  1389. postLog,
  1390. debug), daemon=True)
  1391. sendThreads.append(thr)
  1392. thr.start()
  1393. return 0
  1394. def sendPostViaServer(projectVersion: str,
  1395. baseDir: str, session, fromNickname: str, password: str,
  1396. fromDomain: str, fromPort: int,
  1397. toNickname: str, toDomain: str, toPort: int, cc: str,
  1398. httpPrefix: str, content: str, followersOnly: bool,
  1399. attachImageFilename: str, mediaType: str,
  1400. imageDescription: str, useBlurhash: bool,
  1401. cachedWebfingers: {}, personCache: {},
  1402. isArticle: bool, debug=False, inReplyTo=None,
  1403. inReplyToAtomUri=None, subject=None) -> int:
  1404. """Send a post via a proxy (c2s)
  1405. """
  1406. if not session:
  1407. print('WARN: No session for sendPostViaServer')
  1408. return 6
  1409. if toPort:
  1410. if toPort != 80 and toPort != 443:
  1411. if ':' not in fromDomain:
  1412. fromDomain = fromDomain + ':' + str(fromPort)
  1413. handle = httpPrefix + '://' + fromDomain + '/@' + fromNickname
  1414. # lookup the inbox for the To handle
  1415. wfRequest = \
  1416. webfingerHandle(session, handle, httpPrefix, cachedWebfingers,
  1417. fromDomain, projectVersion)
  1418. if not wfRequest:
  1419. if debug:
  1420. print('DEBUG: webfinger failed for ' + handle)
  1421. return 1
  1422. if not isinstance(wfRequest, dict):
  1423. print('WARN: Webfinger for ' + handle + ' did not return a dict. ' +
  1424. str(wfRequest))
  1425. return 1
  1426. postToBox = 'outbox'
  1427. if isArticle:
  1428. postToBox = 'tlblogs'
  1429. # get the actor inbox for the To handle
  1430. (inboxUrl, pubKeyId, pubKey,
  1431. fromPersonId, sharedInbox,
  1432. capabilityAcquisition,
  1433. avatarUrl, displayName) = getPersonBox(baseDir, session, wfRequest,
  1434. personCache,
  1435. projectVersion, httpPrefix,
  1436. fromNickname,
  1437. fromDomain, postToBox)
  1438. if not inboxUrl:
  1439. if debug:
  1440. print('DEBUG: No ' + postToBox + ' was found for ' + handle)
  1441. return 3
  1442. if not fromPersonId:
  1443. if debug:
  1444. print('DEBUG: No actor was found for ' + handle)
  1445. return 4
  1446. # Get the json for the c2s post, not saving anything to file
  1447. # Note that baseDir is set to None
  1448. saveToFile = False
  1449. clientToServer = True
  1450. if toDomain.lower().endswith('public'):
  1451. toPersonId = 'https://www.w3.org/ns/activitystreams#Public'
  1452. fromDomainFull = fromDomain
  1453. if fromPort:
  1454. if fromPort != 80 and fromPort != 443:
  1455. if ':' not in fromDomain:
  1456. fromDomainFull = fromDomain + ':' + str(fromPort)
  1457. cc = httpPrefix + '://' + fromDomainFull + '/users/' + \
  1458. fromNickname + '/followers'
  1459. else:
  1460. if toDomain.lower().endswith('followers') or \
  1461. toDomain.lower().endswith('followersonly'):
  1462. toPersonId = \
  1463. httpPrefix + '://' + \
  1464. fromDomainFull + '/users/' + fromNickname + '/followers'
  1465. else:
  1466. toDomainFull = toDomain
  1467. if toPort:
  1468. if toPort != 80 and toPort != 443:
  1469. if ':' not in toDomain:
  1470. toDomainFull = toDomain + ':' + str(toPort)
  1471. toPersonId = httpPrefix + '://' + toDomainFull + \
  1472. '/users/' + toNickname
  1473. postJsonObject = \
  1474. createPostBase(baseDir,
  1475. fromNickname, fromDomain, fromPort,
  1476. toPersonId, cc, httpPrefix, content,
  1477. followersOnly, saveToFile, clientToServer,
  1478. attachImageFilename, mediaType,
  1479. imageDescription, useBlurhash,
  1480. False, isArticle, inReplyTo,
  1481. inReplyToAtomUri, subject,
  1482. False, None, None, None)
  1483. authHeader = createBasicAuthHeader(fromNickname, password)
  1484. if attachImageFilename:
  1485. headers = {
  1486. 'host': fromDomain,
  1487. 'Authorization': authHeader
  1488. }
  1489. postResult = \
  1490. postImage(session, attachImageFilename, [],
  1491. inboxUrl, headers, "inbox:write")
  1492. if not postResult:
  1493. if debug:
  1494. print('DEBUG: Failed to upload image')
  1495. # return 9
  1496. headers = {
  1497. 'host': fromDomain,
  1498. 'Content-type': 'application/json',
  1499. 'Authorization': authHeader
  1500. }
  1501. postResult = \
  1502. postJsonString(session, json.dumps(postJsonObject), [],
  1503. inboxUrl, headers, "inbox:write", debug)
  1504. if not postResult:
  1505. if debug:
  1506. print('DEBUG: POST failed for c2s to '+inboxUrl)
  1507. return 5
  1508. if debug:
  1509. print('DEBUG: c2s POST success')
  1510. return 0
  1511. def groupFollowersByDomain(baseDir: str, nickname: str, domain: str) -> {}:
  1512. """Returns a dictionary with followers grouped by domain
  1513. """
  1514. handle = nickname + '@' + domain
  1515. followersFilename = baseDir + '/accounts/' + handle + '/followers.txt'
  1516. if not os.path.isfile(followersFilename):
  1517. return None
  1518. grouped = {}
  1519. with open(followersFilename, "r") as f:
  1520. for followerHandle in f:
  1521. if '@' in followerHandle:
  1522. fHandle = \
  1523. followerHandle.strip().replace('\n', '').replace('\r', '')
  1524. followerDomain = fHandle.split('@')[1]
  1525. if not grouped.get(followerDomain):
  1526. grouped[followerDomain] = [fHandle]
  1527. else:
  1528. grouped[followerDomain].append(fHandle)
  1529. return grouped
  1530. def addFollowersToPublicPost(postJsonObject: {}) -> None:
  1531. """Adds followers entry to cc if it doesn't exist
  1532. """
  1533. if not postJsonObject.get('actor'):
  1534. return
  1535. if isinstance(postJsonObject['object'], str):
  1536. if not postJsonObject.get('to'):
  1537. return
  1538. if len(postJsonObject['to']) > 1:
  1539. return
  1540. if len(postJsonObject['to']) == 0:
  1541. return
  1542. if not postJsonObject['to'][0].endswith('#Public'):
  1543. return
  1544. if postJsonObject.get('cc'):
  1545. return
  1546. postJsonObject['cc'] = postJsonObject['actor'] + '/followers'
  1547. elif isinstance(postJsonObject['object'], dict):
  1548. if not postJsonObject['object'].get('to'):
  1549. return
  1550. if len(postJsonObject['object']['to']) > 1:
  1551. return
  1552. if len(postJsonObject['object']['to']) == 0:
  1553. return
  1554. if not postJsonObject['object']['to'][0].endswith('#Public'):
  1555. return
  1556. if postJsonObject['object'].get('cc'):
  1557. return
  1558. postJsonObject['object']['cc'] = postJsonObject['actor'] + '/followers'
  1559. def sendSignedJson(postJsonObject: {}, session, baseDir: str,
  1560. nickname: str, domain: str, port: int,
  1561. toNickname: str, toDomain: str, toPort: int, cc: str,
  1562. httpPrefix: str, saveToFile: bool, clientToServer: bool,
  1563. federationList: [],
  1564. sendThreads: [], postLog: [], cachedWebfingers: {},
  1565. personCache: {}, debug: bool, projectVersion: str) -> int:
  1566. """Sends a signed json object to an inbox/outbox
  1567. """
  1568. if debug:
  1569. print('DEBUG: sendSignedJson start')
  1570. if not session:
  1571. print('WARN: No session specified for sendSignedJson')
  1572. return 8
  1573. withDigest = True
  1574. if toDomain.endswith('.onion') or toDomain.endswith('.i2p'):
  1575. httpPrefix = 'http'
  1576. # sharedInbox = False
  1577. if toNickname == 'inbox':
  1578. # shared inbox actor on @domain@domain
  1579. toNickname = toDomain
  1580. # sharedInbox = True
  1581. if toPort:
  1582. if toPort != 80 and toPort != 443:
  1583. if ':' not in toDomain:
  1584. toDomain = toDomain + ':' + str(toPort)
  1585. toDomainUrl = httpPrefix + '://' + toDomain
  1586. if not siteIsActive(toDomainUrl):
  1587. print('Domain is inactive: ' + toDomainUrl)
  1588. return 9
  1589. print('Domain is active: ' + toDomainUrl)
  1590. handleBase = toDomainUrl + '/@'
  1591. if toNickname:
  1592. handle = handleBase + toNickname
  1593. else:
  1594. singleUserInstanceNickname = 'dev'
  1595. handle = handleBase + singleUserInstanceNickname
  1596. if debug:
  1597. print('DEBUG: handle - ' + handle + ' toPort ' + str(toPort))
  1598. # lookup the inbox for the To handle
  1599. wfRequest = webfingerHandle(session, handle, httpPrefix, cachedWebfingers,
  1600. domain, projectVersion)
  1601. if not wfRequest:
  1602. if debug:
  1603. print('DEBUG: webfinger for ' + handle + ' failed')
  1604. return 1
  1605. if not isinstance(wfRequest, dict):
  1606. print('WARN: Webfinger for ' + handle + ' did not return a dict. ' +
  1607. str(wfRequest))
  1608. return 1
  1609. if wfRequest.get('errors'):
  1610. if debug:
  1611. print('DEBUG: webfinger for ' + handle +
  1612. ' failed with errors ' + str(wfRequest['errors']))
  1613. if not clientToServer:
  1614. postToBox = 'inbox'
  1615. else:
  1616. postToBox = 'outbox'
  1617. # get the actor inbox/outbox/capabilities for the To handle
  1618. (inboxUrl, pubKeyId, pubKey, toPersonId, sharedInboxUrl,
  1619. capabilityAcquisition, avatarUrl,
  1620. displayName) = getPersonBox(baseDir, session, wfRequest,
  1621. personCache,
  1622. projectVersion, httpPrefix,
  1623. nickname, domain, postToBox)
  1624. if nickname == 'capabilities':
  1625. inboxUrl = capabilityAcquisition
  1626. if not capabilityAcquisition:
  1627. return 2
  1628. else:
  1629. print("inboxUrl: " + str(inboxUrl))
  1630. print("toPersonId: " + str(toPersonId))
  1631. print("sharedInboxUrl: " + str(sharedInboxUrl))
  1632. if inboxUrl:
  1633. if inboxUrl.endswith('/actor/inbox'):
  1634. inboxUrl = sharedInboxUrl
  1635. if not inboxUrl:
  1636. if debug:
  1637. print('DEBUG: missing inboxUrl')
  1638. return 3
  1639. if debug:
  1640. print('DEBUG: Sending to endpoint ' + inboxUrl)
  1641. if not pubKey:
  1642. if debug:
  1643. print('DEBUG: missing pubkey')
  1644. return 4
  1645. if not toPersonId:
  1646. if debug:
  1647. print('DEBUG: missing personId')
  1648. return 5
  1649. # sharedInbox and capabilities are optional
  1650. # get the senders private key
  1651. privateKeyPem = getPersonKey(nickname, domain, baseDir, 'private', debug)
  1652. if len(privateKeyPem) == 0:
  1653. if debug:
  1654. print('DEBUG: Private key not found for ' +
  1655. nickname + '@' + domain + ' in ' + baseDir + '/keys/private')
  1656. return 6
  1657. if toDomain not in inboxUrl:
  1658. if debug:
  1659. print('DEBUG: ' + toDomain + ' is not in ' + inboxUrl)
  1660. return 7
  1661. postPath = inboxUrl.split(toDomain, 1)[1]
  1662. addFollowersToPublicPost(postJsonObject)
  1663. if not postJsonObject.get('signature'):
  1664. try:
  1665. signedPostJsonObject = jsonldSign(postJsonObject, privateKeyPem)
  1666. postJsonObject = signedPostJsonObject
  1667. except BaseException:
  1668. print('WARN: failed to JSON-LD sign post')
  1669. pass
  1670. # convert json to string so that there are no
  1671. # subsequent conversions after creating message body digest
  1672. postJsonStr = json.dumps(postJsonObject)
  1673. # construct the http header, including the message body digest
  1674. signatureHeaderJson = \
  1675. createSignedHeader(privateKeyPem, nickname, domain, port,
  1676. toDomain, toPort,
  1677. postPath, httpPrefix, withDigest, postJsonStr)
  1678. # Keep the number of threads being used small
  1679. while len(sendThreads) > 1000:
  1680. print('WARN: Maximum threads reached - killing send thread')
  1681. sendThreads[0].kill()
  1682. sendThreads.pop(0)
  1683. print('WARN: thread killed')
  1684. if debug:
  1685. print('DEBUG: starting thread to send post')
  1686. pprint(postJsonObject)
  1687. thr = \
  1688. threadWithTrace(target=threadSendPost,
  1689. args=(session,
  1690. postJsonStr,
  1691. federationList,
  1692. inboxUrl, baseDir,
  1693. signatureHeaderJson.copy(),
  1694. postLog,
  1695. debug), daemon=True)
  1696. sendThreads.append(thr)
  1697. # thr.start()
  1698. return 0
  1699. def addToField(activityType: str, postJsonObject: {},
  1700. debug: bool) -> ({}, bool):
  1701. """The Follow activity doesn't have a 'to' field and so one
  1702. needs to be added so that activity distribution happens in a consistent way
  1703. Returns true if a 'to' field exists or was added
  1704. """
  1705. if postJsonObject.get('to'):
  1706. return postJsonObject, True
  1707. if debug:
  1708. pprint(postJsonObject)
  1709. print('DEBUG: no "to" field when sending to named addresses 2')
  1710. isSameType = False
  1711. toFieldAdded = False
  1712. if postJsonObject.get('object'):
  1713. if isinstance(postJsonObject['object'], str):
  1714. if postJsonObject.get('type'):
  1715. if postJsonObject['type'] == activityType:
  1716. isSameType = True
  1717. if debug:
  1718. print('DEBUG: "to" field assigned to Follow')
  1719. toAddress = postJsonObject['object']
  1720. if '/statuses/' in toAddress:
  1721. toAddress = toAddress.split('/statuses/')[0]
  1722. postJsonObject['to'] = [toAddress]
  1723. toFieldAdded = True
  1724. elif isinstance(postJsonObject['object'], dict):
  1725. if postJsonObject['object'].get('type'):
  1726. if postJsonObject['object']['type'] == activityType:
  1727. isSameType = True
  1728. if isinstance(postJsonObject['object']['object'], str):
  1729. if debug:
  1730. print('DEBUG: "to" field assigned to Follow')
  1731. toAddress = postJsonObject['object']['object']
  1732. if '/statuses/' in toAddress:
  1733. toAddress = toAddress.split('/statuses/')[0]
  1734. postJsonObject['object']['to'] = [toAddress]
  1735. postJsonObject['to'] = \
  1736. [postJsonObject['object']['object']]
  1737. toFieldAdded = True
  1738. if not isSameType:
  1739. return postJsonObject, True
  1740. if toFieldAdded:
  1741. return postJsonObject, True
  1742. return postJsonObject, False
  1743. def sendToNamedAddresses(session, baseDir: str,
  1744. nickname: str,
  1745. domain: str,
  1746. onionDomain: str, i2pDomain: str, port: int,
  1747. httpPrefix: str, federationList: [],
  1748. sendThreads: [], postLog: [],
  1749. cachedWebfingers: {}, personCache: {},
  1750. postJsonObject: {}, debug: bool,
  1751. projectVersion: str) -> None:
  1752. """sends a post to the specific named addresses in to/cc
  1753. """
  1754. if not session:
  1755. print('WARN: No session for sendToNamedAddresses')
  1756. return
  1757. if not postJsonObject.get('object'):
  1758. return
  1759. if isinstance(postJsonObject['object'], dict):
  1760. isProfileUpdate = False
  1761. # for actor updates there is no 'to' within the object
  1762. if postJsonObject['object'].get('type') and postJsonObject.get('type'):
  1763. if (postJsonObject['type'] == 'Update' and
  1764. (postJsonObject['object']['type'] == 'Person' or
  1765. postJsonObject['object']['type'] == 'Application' or
  1766. postJsonObject['object']['type'] == 'Group' or
  1767. postJsonObject['object']['type'] == 'Service')):
  1768. # use the original object, which has a 'to'
  1769. recipientsObject = postJsonObject
  1770. isProfileUpdate = True
  1771. if not isProfileUpdate:
  1772. if not postJsonObject['object'].get('to'):
  1773. if debug:
  1774. pprint(postJsonObject)
  1775. print('DEBUG: ' +
  1776. 'no "to" field when sending to named addresses')
  1777. if postJsonObject['object'].get('type'):
  1778. if postJsonObject['object']['type'] == 'Follow':
  1779. if isinstance(postJsonObject['object']['object'], str):
  1780. if debug:
  1781. print('DEBUG: "to" field assigned to Follow')
  1782. postJsonObject['object']['to'] = \
  1783. [postJsonObject['object']['object']]
  1784. if not postJsonObject['object'].get('to'):
  1785. return
  1786. recipientsObject = postJsonObject['object']
  1787. else:
  1788. postJsonObject, fieldAdded = \
  1789. addToField('Follow', postJsonObject, debug)
  1790. if not fieldAdded:
  1791. return
  1792. postJsonObject, fieldAdded = addToField('Like', postJsonObject, debug)
  1793. if not fieldAdded:
  1794. return
  1795. recipientsObject = postJsonObject
  1796. recipients = []
  1797. recipientType = ('to', 'cc')
  1798. for rType in recipientType:
  1799. if not recipientsObject.get(rType):
  1800. continue
  1801. if isinstance(recipientsObject[rType], list):
  1802. if debug:
  1803. pprint(recipientsObject)
  1804. print('recipientsObject: ' + str(recipientsObject[rType]))
  1805. for address in recipientsObject[rType]:
  1806. if not address:
  1807. continue
  1808. if '/' not in address:
  1809. continue
  1810. if address.endswith('#Public'):
  1811. continue
  1812. if address.endswith('/followers'):
  1813. continue
  1814. recipients.append(address)
  1815. elif isinstance(recipientsObject[rType], str):
  1816. address = recipientsObject[rType]
  1817. if address:
  1818. if '/' in address:
  1819. if address.endswith('#Public'):
  1820. continue
  1821. if address.endswith('/followers'):
  1822. continue
  1823. recipients.append(address)
  1824. if not recipients:
  1825. if debug:
  1826. print('DEBUG: no individual recipients')
  1827. return
  1828. if debug:
  1829. print('DEBUG: Sending individually addressed posts: ' +
  1830. str(recipients))
  1831. # this is after the message has arrived at the server
  1832. clientToServer = False
  1833. for address in recipients:
  1834. toNickname = getNicknameFromActor(address)
  1835. if not toNickname:
  1836. continue
  1837. toDomain, toPort = getDomainFromActor(address)
  1838. if not toDomain:
  1839. continue
  1840. if debug:
  1841. domainFull = domain
  1842. if port:
  1843. if port != 80 and port != 443:
  1844. if ':' not in domain:
  1845. domainFull = domain + ':' + str(port)
  1846. toDomainFull = toDomain
  1847. if toPort:
  1848. if toPort != 80 and toPort != 443:
  1849. if ':' not in toDomain:
  1850. toDomainFull = toDomain + ':' + str(toPort)
  1851. print('DEBUG: Post sending s2s: ' + nickname + '@' + domainFull +
  1852. ' to ' + toNickname + '@' + toDomainFull)
  1853. # if we have an alt onion domain and we are sending to
  1854. # another onion domain then switch the clearnet
  1855. # domain for the onion one
  1856. fromDomain = domain
  1857. fromHttpPrefix = httpPrefix
  1858. if onionDomain:
  1859. if toDomain.endswith('.onion'):
  1860. fromDomain = onionDomain
  1861. fromHttpPrefix = 'http'
  1862. elif i2pDomain:
  1863. if toDomain.endswith('.i2p'):
  1864. fromDomain = i2pDomain
  1865. fromHttpPrefix = 'http'
  1866. cc = []
  1867. sendSignedJson(postJsonObject, session, baseDir,
  1868. nickname, fromDomain, port,
  1869. toNickname, toDomain, toPort,
  1870. cc, fromHttpPrefix, True, clientToServer,
  1871. federationList,
  1872. sendThreads, postLog, cachedWebfingers,
  1873. personCache, debug, projectVersion)
  1874. def hasSharedInbox(session, httpPrefix: str, domain: str) -> bool:
  1875. """Returns true if the given domain has a shared inbox
  1876. """
  1877. wfRequest = webfingerHandle(session, domain + '@' + domain,
  1878. httpPrefix, {},
  1879. None, __version__)
  1880. if wfRequest:
  1881. if isinstance(wfRequest, dict):
  1882. if not wfRequest.get('errors'):
  1883. return True
  1884. return False
  1885. def sendToFollowers(session, baseDir: str,
  1886. nickname: str,
  1887. domain: str,
  1888. onionDomain: str, i2pDomain: str, port: int,
  1889. httpPrefix: str, federationList: [],
  1890. sendThreads: [], postLog: [],
  1891. cachedWebfingers: {}, personCache: {},
  1892. postJsonObject: {}, debug: bool,
  1893. projectVersion: str) -> None:
  1894. """sends a post to the followers of the given nickname
  1895. """
  1896. print('sendToFollowers')
  1897. if not session:
  1898. print('WARN: No session for sendToFollowers')
  1899. return
  1900. if not postIsAddressedToFollowers(baseDir, nickname, domain,
  1901. port, httpPrefix,
  1902. postJsonObject):
  1903. if debug:
  1904. print('Post is not addressed to followers')
  1905. return
  1906. print('Post is addressed to followers')
  1907. grouped = groupFollowersByDomain(baseDir, nickname, domain)
  1908. if not grouped:
  1909. if debug:
  1910. print('Post to followers did not resolve any domains')
  1911. return
  1912. print('Post to followers resolved domains')
  1913. print(str(grouped))
  1914. # this is after the message has arrived at the server
  1915. clientToServer = False
  1916. # for each instance
  1917. for followerDomain, followerHandles in grouped.items():
  1918. if debug:
  1919. print('DEBUG: follower handles for ' + followerDomain)
  1920. pprint(followerHandles)
  1921. # check that the follower's domain is active
  1922. followerDomainUrl = httpPrefix + '://' + followerDomain
  1923. if not siteIsActive(followerDomainUrl):
  1924. print('Domain is inactive: ' + followerDomainUrl)
  1925. continue
  1926. print('Domain is active: ' + followerDomainUrl)
  1927. withSharedInbox = hasSharedInbox(session, httpPrefix, followerDomain)
  1928. if debug:
  1929. if withSharedInbox:
  1930. print(followerDomain + ' has shared inbox')
  1931. else:
  1932. print(followerDomain + ' does not have a shared inbox')
  1933. toPort = port
  1934. index = 0
  1935. toDomain = followerHandles[index].split('@')[1]
  1936. if ':' in toDomain:
  1937. toPort = toDomain.split(':')[1]
  1938. toDomain = toDomain.split(':')[0]
  1939. cc = ''
  1940. # if we are sending to an onion domain and we
  1941. # have an alt onion domain then use the alt
  1942. fromDomain = domain
  1943. fromHttpPrefix = httpPrefix
  1944. if onionDomain:
  1945. if toDomain.endswith('.onion'):
  1946. fromDomain = onionDomain
  1947. fromHttpPrefix = 'http'
  1948. elif i2pDomain:
  1949. if toDomain.endswith('.i2p'):
  1950. fromDomain = i2pDomain
  1951. fromHttpPrefix = 'http'
  1952. if withSharedInbox:
  1953. toNickname = followerHandles[index].split('@')[0]
  1954. # if there are more than one followers on the domain
  1955. # then send the post to the shared inbox
  1956. if len(followerHandles) > 1:
  1957. toNickname = 'inbox'
  1958. if toNickname != 'inbox' and postJsonObject.get('type'):
  1959. if postJsonObject['type'] == 'Update':
  1960. if postJsonObject.get('object'):
  1961. if isinstance(postJsonObject['object'], dict):
  1962. if postJsonObject['object'].get('type'):
  1963. typ = postJsonObject['object']['type']
  1964. if typ == 'Person' or \
  1965. typ == 'Application' or \
  1966. typ == 'Group' or \
  1967. typ == 'Service':
  1968. print('Sending profile update to ' +
  1969. 'shared inbox of ' + toDomain)
  1970. toNickname = 'inbox'
  1971. if debug:
  1972. print('DEBUG: Sending from ' + nickname + '@' + domain +
  1973. ' to ' + toNickname + '@' + toDomain)
  1974. sendSignedJson(postJsonObject, session, baseDir,
  1975. nickname, fromDomain, port,
  1976. toNickname, toDomain, toPort,
  1977. cc, fromHttpPrefix, True, clientToServer,
  1978. federationList,
  1979. sendThreads, postLog, cachedWebfingers,
  1980. personCache, debug, projectVersion)
  1981. else:
  1982. # send to individual followers without using a shared inbox
  1983. for handle in followerHandles:
  1984. if debug:
  1985. print('DEBUG: Sending to ' + handle)
  1986. toNickname = handle.split('@')[0]
  1987. if debug:
  1988. if postJsonObject['type'] != 'Update':
  1989. print('DEBUG: Sending from ' +
  1990. nickname + '@' + domain + ' to ' +
  1991. toNickname + '@' + toDomain)
  1992. else:
  1993. print('DEBUG: Sending profile update from ' +
  1994. nickname + '@' + domain + ' to ' +
  1995. toNickname + '@' + toDomain)
  1996. sendSignedJson(postJsonObject, session, baseDir,
  1997. nickname, fromDomain, port,
  1998. toNickname, toDomain, toPort,
  1999. cc, fromHttpPrefix, True, clientToServer,
  2000. federationList,
  2001. sendThreads, postLog, cachedWebfingers,
  2002. personCache, debug, projectVersion)
  2003. time.sleep(4)
  2004. if debug:
  2005. print('DEBUG: End of sendToFollowers')
  2006. def sendToFollowersThread(session, baseDir: str,
  2007. nickname: str,
  2008. domain: str,
  2009. onionDomain: str, i2pDomain: str, port: int,
  2010. httpPrefix: str, federationList: [],
  2011. sendThreads: [], postLog: [],
  2012. cachedWebfingers: {}, personCache: {},
  2013. postJsonObject: {}, debug: bool,
  2014. projectVersion: str):
  2015. """Returns a thread used to send a post to followers
  2016. """
  2017. sendThread = \
  2018. threadWithTrace(target=sendToFollowers,
  2019. args=(session, baseDir,
  2020. nickname, domain,
  2021. onionDomain, i2pDomain, port,
  2022. httpPrefix, federationList,
  2023. sendThreads, postLog,
  2024. cachedWebfingers, personCache,
  2025. postJsonObject.copy(), debug,
  2026. projectVersion), daemon=True)
  2027. try:
  2028. sendThread.start()
  2029. except SocketError as e:
  2030. print('WARN: socket error while starting ' +
  2031. 'thread to send to followers. ' + str(e))
  2032. return None
  2033. except ValueError as e:
  2034. print('WARN: error while starting ' +
  2035. 'thread to send to followers. ' + str(e))
  2036. return None
  2037. return sendThread
  2038. def createInbox(recentPostsCache: {},
  2039. session, baseDir: str, nickname: str, domain: str, port: int,
  2040. httpPrefix: str, itemsPerPage: int, headerOnly: bool,
  2041. ocapAlways: bool, pageNumber=None) -> {}:
  2042. return createBoxIndexed(recentPostsCache,
  2043. session, baseDir, 'inbox',
  2044. nickname, domain, port, httpPrefix,
  2045. itemsPerPage, headerOnly, True,
  2046. ocapAlways, pageNumber)
  2047. def createBookmarksTimeline(session, baseDir: str, nickname: str, domain: str,
  2048. port: int, httpPrefix: str, itemsPerPage: int,
  2049. headerOnly: bool, ocapAlways: bool,
  2050. pageNumber=None) -> {}:
  2051. return createBoxIndexed({}, session, baseDir, 'tlbookmarks',
  2052. nickname, domain,
  2053. port, httpPrefix, itemsPerPage, headerOnly,
  2054. True, ocapAlways, pageNumber)
  2055. def createDMTimeline(session, baseDir: str, nickname: str, domain: str,
  2056. port: int, httpPrefix: str, itemsPerPage: int,
  2057. headerOnly: bool, ocapAlways: bool,
  2058. pageNumber=None) -> {}:
  2059. return createBoxIndexed({}, session, baseDir, 'dm', nickname,
  2060. domain, port, httpPrefix, itemsPerPage,
  2061. headerOnly, True, ocapAlways, pageNumber)
  2062. def createRepliesTimeline(session, baseDir: str, nickname: str, domain: str,
  2063. port: int, httpPrefix: str, itemsPerPage: int,
  2064. headerOnly: bool, ocapAlways: bool,
  2065. pageNumber=None) -> {}:
  2066. return createBoxIndexed({}, session, baseDir, 'tlreplies',
  2067. nickname, domain, port, httpPrefix,
  2068. itemsPerPage, headerOnly, True,
  2069. ocapAlways, pageNumber)
  2070. def createBlogsTimeline(session, baseDir: str, nickname: str, domain: str,
  2071. port: int, httpPrefix: str, itemsPerPage: int,
  2072. headerOnly: bool, ocapAlways: bool,
  2073. pageNumber=None) -> {}:
  2074. return createBoxIndexed({}, session, baseDir, 'tlblogs', nickname,
  2075. domain, port, httpPrefix,
  2076. itemsPerPage, headerOnly, True,
  2077. ocapAlways, pageNumber)
  2078. def createMediaTimeline(session, baseDir: str, nickname: str, domain: str,
  2079. port: int, httpPrefix: str, itemsPerPage: int,
  2080. headerOnly: bool, ocapAlways: bool,
  2081. pageNumber=None) -> {}:
  2082. return createBoxIndexed({}, session, baseDir, 'tlmedia', nickname,
  2083. domain, port, httpPrefix,
  2084. itemsPerPage, headerOnly, True,
  2085. ocapAlways, pageNumber)
  2086. def createOutbox(session, baseDir: str, nickname: str, domain: str,
  2087. port: int, httpPrefix: str,
  2088. itemsPerPage: int, headerOnly: bool, authorized: bool,
  2089. pageNumber=None) -> {}:
  2090. return createBoxIndexed({}, session, baseDir, 'outbox',
  2091. nickname, domain, port, httpPrefix,
  2092. itemsPerPage, headerOnly, authorized,
  2093. False, pageNumber)
  2094. def createModeration(baseDir: str, nickname: str, domain: str, port: int,
  2095. httpPrefix: str, itemsPerPage: int, headerOnly: bool,
  2096. ocapAlways: bool, pageNumber=None) -> {}:
  2097. boxDir = createPersonDir(nickname, domain, baseDir, 'inbox')
  2098. boxname = 'moderation'
  2099. if port:
  2100. if port != 80 and port != 443:
  2101. if ':' not in domain:
  2102. domain = domain + ':' + str(port)
  2103. if not pageNumber:
  2104. pageNumber = 1
  2105. pageStr = '?page=' + str(pageNumber)
  2106. boxUrl = httpPrefix+'://'+domain+'/users/'+nickname+'/'+boxname
  2107. boxHeader = {
  2108. '@context': 'https://www.w3.org/ns/activitystreams',
  2109. 'first': boxUrl+'?page=true',
  2110. 'id': boxUrl,
  2111. 'last': boxUrl+'?page=true',
  2112. 'totalItems': 0,
  2113. 'type': 'OrderedCollection'
  2114. }
  2115. boxItems = {
  2116. '@context': 'https://www.w3.org/ns/activitystreams',
  2117. 'id': boxUrl+pageStr,
  2118. 'orderedItems': [
  2119. ],
  2120. 'partOf': boxUrl,
  2121. 'type': 'OrderedCollectionPage'
  2122. }
  2123. if isModerator(baseDir, nickname):
  2124. moderationIndexFile = baseDir + '/accounts/moderation.txt'
  2125. if os.path.isfile(moderationIndexFile):
  2126. with open(moderationIndexFile, "r") as f:
  2127. lines = f.readlines()
  2128. boxHeader['totalItems'] = len(lines)
  2129. if headerOnly:
  2130. return boxHeader
  2131. pageLines = []
  2132. if len(lines) > 0:
  2133. endLineNumber = len(lines) - 1 - int(itemsPerPage * pageNumber)
  2134. if endLineNumber < 0:
  2135. endLineNumber = 0
  2136. startLineNumber = \
  2137. len(lines) - 1 - int(itemsPerPage * (pageNumber - 1))
  2138. if startLineNumber < 0:
  2139. startLineNumber = 0
  2140. lineNumber = startLineNumber
  2141. while lineNumber >= endLineNumber:
  2142. pageLines.append(lines[lineNumber].strip('\n').strip('\r'))
  2143. lineNumber -= 1
  2144. for postUrl in pageLines:
  2145. postFilename = \
  2146. boxDir + '/' + postUrl.replace('/', '#') + '.json'
  2147. if os.path.isfile(postFilename):
  2148. postJsonObject = loadJson(postFilename)
  2149. if postJsonObject:
  2150. boxItems['orderedItems'].append(postJsonObject)
  2151. if headerOnly:
  2152. return boxHeader
  2153. return boxItems
  2154. def getStatusNumberFromPostFilename(filename) -> int:
  2155. """Gets the status number from a post filename
  2156. eg. https:##testdomain.com:8085#users#testuser567#
  2157. statuses#1562958506952068.json
  2158. returns 156295850695206
  2159. """
  2160. if '#statuses#' not in filename:
  2161. return None
  2162. return int(filename.split('#')[-1].replace('.json', ''))
  2163. def isDM(postJsonObject: {}) -> bool:
  2164. """Returns true if the given post is a DM
  2165. """
  2166. if postJsonObject['type'] != 'Create':
  2167. return False
  2168. if not postJsonObject.get('object'):
  2169. return False
  2170. if not isinstance(postJsonObject['object'], dict):
  2171. return False
  2172. if postJsonObject['object']['type'] != 'Note' and \
  2173. postJsonObject['object']['type'] != 'Patch' and \
  2174. postJsonObject['object']['type'] != 'Article':
  2175. return False
  2176. if postJsonObject['object'].get('moderationStatus'):
  2177. return False
  2178. fields = ('to', 'cc')
  2179. for f in fields:
  2180. if not postJsonObject['object'].get(f):
  2181. continue
  2182. for toAddress in postJsonObject['object'][f]:
  2183. if toAddress.endswith('#Public'):
  2184. return False
  2185. if toAddress.endswith('followers'):
  2186. return False
  2187. return True
  2188. def isImageMedia(session, baseDir: str, httpPrefix: str,
  2189. nickname: str, domain: str,
  2190. postJsonObject: {}, translate: {}) -> bool:
  2191. """Returns true if the given post has attached image media
  2192. """
  2193. if postJsonObject['type'] == 'Announce':
  2194. postJsonAnnounce = \
  2195. downloadAnnounce(session, baseDir, httpPrefix,
  2196. nickname, domain, postJsonObject,
  2197. __version__, translate)
  2198. if postJsonAnnounce:
  2199. postJsonObject = postJsonAnnounce
  2200. if postJsonObject['type'] != 'Create':
  2201. return False
  2202. if not postJsonObject.get('object'):
  2203. return False
  2204. if not isinstance(postJsonObject['object'], dict):
  2205. return False
  2206. if postJsonObject['object'].get('moderationStatus'):
  2207. return False
  2208. if postJsonObject['object']['type'] != 'Note' and \
  2209. postJsonObject['object']['type'] != 'Article':
  2210. return False
  2211. if not postJsonObject['object'].get('attachment'):
  2212. return False
  2213. if not isinstance(postJsonObject['object']['attachment'], list):
  2214. return False
  2215. for attach in postJsonObject['object']['attachment']:
  2216. if attach.get('mediaType') and attach.get('url'):
  2217. if attach['mediaType'].startswith('image/') or \
  2218. attach['mediaType'].startswith('audio/') or \
  2219. attach['mediaType'].startswith('video/'):
  2220. return True
  2221. return False
  2222. def isReply(postJsonObject: {}, actor: str) -> bool:
  2223. """Returns true if the given post is a reply to the given actor
  2224. """
  2225. if postJsonObject['type'] != 'Create':
  2226. return False
  2227. if not postJsonObject.get('object'):
  2228. return False
  2229. if not isinstance(postJsonObject['object'], dict):
  2230. return False
  2231. if postJsonObject['object'].get('moderationStatus'):
  2232. return False
  2233. if postJsonObject['object']['type'] != 'Note' and \
  2234. postJsonObject['object']['type'] != 'Article':
  2235. return False
  2236. if postJsonObject['object'].get('inReplyTo'):
  2237. if postJsonObject['object']['inReplyTo'].startswith(actor):
  2238. return True
  2239. if not postJsonObject['object'].get('tag'):
  2240. return False
  2241. if not isinstance(postJsonObject['object']['tag'], list):
  2242. return False
  2243. for tag in postJsonObject['object']['tag']:
  2244. if not tag.get('type'):
  2245. continue
  2246. if tag['type'] == 'Mention':
  2247. if not tag.get('href'):
  2248. continue
  2249. if actor in tag['href']:
  2250. return True
  2251. return False
  2252. def createBoxIndex(boxDir: str, postsInBoxDict: {}) -> int:
  2253. """ Creates an index for the given box
  2254. """
  2255. postsCtr = 0
  2256. postsInPersonInbox = os.scandir(boxDir)
  2257. for postFilename in postsInPersonInbox:
  2258. postFilename = postFilename.name
  2259. if not postFilename.endswith('.json'):
  2260. continue
  2261. # extract the status number
  2262. statusNumber = getStatusNumberFromPostFilename(postFilename)
  2263. if statusNumber:
  2264. postsInBoxDict[statusNumber] = os.path.join(boxDir, postFilename)
  2265. postsCtr += 1
  2266. return postsCtr
  2267. def createSharedInboxIndex(baseDir: str, sharedBoxDir: str,
  2268. postsInBoxDict: {}, postsCtr: int,
  2269. nickname: str, domain: str,
  2270. ocapAlways: bool) -> int:
  2271. """ Creates an index for the given shared inbox
  2272. """
  2273. handle = nickname + '@' + domain
  2274. followingFilename = baseDir + '/accounts/' + handle + '/following.txt'
  2275. postsInSharedInbox = os.scandir(sharedBoxDir)
  2276. followingHandles = None
  2277. for postFilename in postsInSharedInbox:
  2278. postFilename = postFilename.name
  2279. if not postFilename.endswith('.json'):
  2280. continue
  2281. statusNumber = getStatusNumberFromPostFilename(postFilename)
  2282. if not statusNumber:
  2283. continue
  2284. sharedInboxFilename = os.path.join(sharedBoxDir, postFilename)
  2285. # get the actor from the shared post
  2286. postJsonObject = loadJson(sharedInboxFilename, 0)
  2287. if not postJsonObject:
  2288. print('WARN: json load exception createSharedInboxIndex')
  2289. continue
  2290. actorNickname = getNicknameFromActor(postJsonObject['actor'])
  2291. if not actorNickname:
  2292. continue
  2293. actorDomain, actorPort = getDomainFromActor(postJsonObject['actor'])
  2294. if not actorDomain:
  2295. continue
  2296. # is the actor followed by this account?
  2297. if not followingHandles:
  2298. with open(followingFilename, 'r') as followingFile:
  2299. followingHandles = followingFile.read()
  2300. if actorNickname + '@' + actorDomain not in followingHandles:
  2301. continue
  2302. if ocapAlways:
  2303. capsList = None
  2304. # Note: should this be in the Create or the object of a post?
  2305. if postJsonObject.get('capability'):
  2306. if isinstance(postJsonObject['capability'], list):
  2307. capsList = postJsonObject['capability']
  2308. # Have capabilities been granted for the sender?
  2309. ocapFilename = \
  2310. baseDir + '/accounts/' + handle + '/ocap/granted/' + \
  2311. postJsonObject['actor'].replace('/', '#') + '.json'
  2312. if not os.path.isfile(ocapFilename):
  2313. continue
  2314. # read the capabilities id
  2315. ocapJson = loadJson(ocapFilename, 0)
  2316. if not ocapJson:
  2317. print('WARN: json load exception createSharedInboxIndex')
  2318. else:
  2319. if ocapJson.get('id'):
  2320. if ocapJson['id'] in capsList:
  2321. postsInBoxDict[statusNumber] = sharedInboxFilename
  2322. postsCtr += 1
  2323. else:
  2324. postsInBoxDict[statusNumber] = sharedInboxFilename
  2325. postsCtr += 1
  2326. return postsCtr
  2327. def addPostStringToTimeline(postStr: str, boxname: str,
  2328. postsInBox: [], boxActor: str) -> bool:
  2329. """ is this a valid timeline post?
  2330. """
  2331. # must be a recognized ActivityPub type
  2332. if ('"Note"' in postStr or
  2333. '"Article"' in postStr or
  2334. '"Patch"' in postStr or
  2335. '"Announce"' in postStr or
  2336. ('"Question"' in postStr and
  2337. ('"Create"' in postStr or '"Update"' in postStr))):
  2338. if boxname == 'dm':
  2339. if '#Public' in postStr or '/followers' in postStr:
  2340. return False
  2341. elif boxname == 'tlreplies':
  2342. if boxActor not in postStr:
  2343. return False
  2344. elif boxname == 'tlblogs':
  2345. if '"Create"' not in postStr:
  2346. return False
  2347. if '"Article"' not in postStr:
  2348. return False
  2349. elif boxname == 'tlmedia':
  2350. if '"Create"' in postStr:
  2351. if 'mediaType' not in postStr or 'image/' not in postStr:
  2352. return False
  2353. # add the post to the dictionary
  2354. postsInBox.append(postStr)
  2355. return True
  2356. return False
  2357. def addPostToTimeline(filePath: str, boxname: str,
  2358. postsInBox: [], boxActor: str) -> bool:
  2359. """ Reads a post from file and decides whether it is valid
  2360. """
  2361. with open(filePath, 'r') as postFile:
  2362. postStr = postFile.read()
  2363. return addPostStringToTimeline(postStr, boxname, postsInBox, boxActor)
  2364. return False
  2365. def createBoxIndexed(recentPostsCache: {},
  2366. session, baseDir: str, boxname: str,
  2367. nickname: str, domain: str, port: int, httpPrefix: str,
  2368. itemsPerPage: int, headerOnly: bool, authorized: bool,
  2369. ocapAlways: bool, pageNumber=None) -> {}:
  2370. """Constructs the box feed for a person with the given nickname
  2371. """
  2372. if not authorized or not pageNumber:
  2373. pageNumber = 1
  2374. if boxname != 'inbox' and boxname != 'dm' and \
  2375. boxname != 'tlreplies' and boxname != 'tlmedia' and \
  2376. boxname != 'tlblogs' and \
  2377. boxname != 'outbox' and boxname != 'tlbookmarks' and \
  2378. boxname != 'bookmarks':
  2379. return None
  2380. # bookmarks timeline is like the inbox but has its own separate index
  2381. indexBoxName = boxname
  2382. if boxname == "tlbookmarks":
  2383. boxname = "bookmarks"
  2384. indexBoxName = boxname
  2385. if port:
  2386. if port != 80 and port != 443:
  2387. if ':' not in domain:
  2388. domain = domain + ':' + str(port)
  2389. boxActor = httpPrefix + '://' + domain + '/users/' + nickname
  2390. pageStr = '?page=true'
  2391. if pageNumber:
  2392. if pageNumber < 1:
  2393. pageNumber = 1
  2394. try:
  2395. pageStr = '?page=' + str(pageNumber)
  2396. except BaseException:
  2397. pass
  2398. boxUrl = httpPrefix + '://' + domain + '/users/' + nickname + '/' + boxname
  2399. boxHeader = {
  2400. '@context': 'https://www.w3.org/ns/activitystreams',
  2401. 'first': boxUrl + '?page=true',
  2402. 'id': boxUrl,
  2403. 'last': boxUrl + '?page=true',
  2404. 'totalItems': 0,
  2405. 'type': 'OrderedCollection'
  2406. }
  2407. boxItems = {
  2408. '@context': 'https://www.w3.org/ns/activitystreams',
  2409. 'id': boxUrl + pageStr,
  2410. 'orderedItems': [
  2411. ],
  2412. 'partOf': boxUrl,
  2413. 'type': 'OrderedCollectionPage'
  2414. }
  2415. postsInBox = []
  2416. indexFilename = \
  2417. baseDir + '/accounts/' + nickname + '@' + domain + \
  2418. '/' + indexBoxName + '.index'
  2419. postsCtr = 0
  2420. if os.path.isfile(indexFilename):
  2421. maxPostCtr = itemsPerPage * pageNumber
  2422. with open(indexFilename, 'r') as indexFile:
  2423. while postsCtr < maxPostCtr:
  2424. postFilename = indexFile.readline()
  2425. if not postFilename:
  2426. break
  2427. # Skip through any posts previous to the current page
  2428. if postsCtr < int((pageNumber - 1) * itemsPerPage):
  2429. postsCtr += 1
  2430. continue
  2431. # if this is a full path then remove the directories
  2432. if '/' in postFilename:
  2433. postFilename = postFilename.split('/')[-1]
  2434. # filename of the post without any extension or path
  2435. # This should also correspond to any index entry in
  2436. # the posts cache
  2437. postUrl = \
  2438. postFilename.replace('\n', '').replace('\r', '')
  2439. postUrl = postUrl.replace('.json', '').strip()
  2440. # is the post cached in memory?
  2441. if recentPostsCache.get('index'):
  2442. if postUrl in recentPostsCache['index']:
  2443. if recentPostsCache['json'].get(postUrl):
  2444. url = recentPostsCache['json'][postUrl]
  2445. addPostStringToTimeline(url,
  2446. boxname, postsInBox,
  2447. boxActor)
  2448. postsCtr += 1
  2449. continue
  2450. # read the post from file
  2451. fullPostFilename = \
  2452. locatePost(baseDir, nickname,
  2453. domain, postUrl, False)
  2454. if fullPostFilename:
  2455. addPostToTimeline(fullPostFilename, boxname,
  2456. postsInBox, boxActor)
  2457. else:
  2458. print('WARN: unable to locate post ' + postUrl)
  2459. postsCtr += 1
  2460. # Generate first and last entries within header
  2461. if postsCtr > 0:
  2462. lastPage = int(postsCtr / itemsPerPage)
  2463. if lastPage < 1:
  2464. lastPage = 1
  2465. boxHeader['last'] = \
  2466. httpPrefix + '://' + domain + '/users/' + \
  2467. nickname + '/' + boxname + '?page=' + str(lastPage)
  2468. if headerOnly:
  2469. boxHeader['totalItems'] = len(postsInBox)
  2470. prevPageStr = 'true'
  2471. if pageNumber > 1:
  2472. prevPageStr = str(pageNumber - 1)
  2473. boxHeader['prev'] = \
  2474. httpPrefix + '://' + domain + '/users/' + \
  2475. nickname + '/' + boxname + '?page=' + prevPageStr
  2476. nextPageStr = str(pageNumber + 1)
  2477. boxHeader['next'] = \
  2478. httpPrefix + '://' + domain + '/users/' + \
  2479. nickname + '/' + boxname + '?page=' + nextPageStr
  2480. return boxHeader
  2481. for postStr in postsInBox:
  2482. p = None
  2483. try:
  2484. p = json.loads(postStr)
  2485. except BaseException:
  2486. continue
  2487. # remove any capability so that it's not displayed
  2488. if p.get('capability'):
  2489. del p['capability']
  2490. # Don't show likes, replies or shares (announces) to
  2491. # unauthorized viewers
  2492. if not authorized:
  2493. if p.get('object'):
  2494. if isinstance(p['object'], dict):
  2495. if p['object'].get('likes'):
  2496. p['likes'] = {'items': []}
  2497. if p['object'].get('replies'):
  2498. p['replies'] = {}
  2499. if p['object'].get('shares'):
  2500. p['shares'] = {}
  2501. if p['object'].get('bookmarks'):
  2502. p['bookmarks'] = {}
  2503. boxItems['orderedItems'].append(p)
  2504. return boxItems
  2505. def expireCache(baseDir: str, personCache: {},
  2506. httpPrefix: str, archiveDir: str,
  2507. recentPostsCache: {},
  2508. maxPostsInBox=32000):
  2509. """Thread used to expire actors from the cache and archive old posts
  2510. """
  2511. while True:
  2512. # once per day
  2513. time.sleep(60 * 60 * 24)
  2514. expirePersonCache(baseDir, personCache)
  2515. archivePosts(baseDir, httpPrefix, archiveDir, recentPostsCache,
  2516. maxPostsInBox)
  2517. def archivePosts(baseDir: str, httpPrefix: str, archiveDir: str,
  2518. recentPostsCache: {},
  2519. maxPostsInBox=32000) -> None:
  2520. """Archives posts for all accounts
  2521. """
  2522. if archiveDir:
  2523. if not os.path.isdir(archiveDir):
  2524. os.mkdir(archiveDir)
  2525. if archiveDir:
  2526. if not os.path.isdir(archiveDir + '/accounts'):
  2527. os.mkdir(archiveDir + '/accounts')
  2528. for subdir, dirs, files in os.walk(baseDir + '/accounts'):
  2529. for handle in dirs:
  2530. if '@' in handle:
  2531. nickname = handle.split('@')[0]
  2532. domain = handle.split('@')[1]
  2533. archiveSubdir = None
  2534. if archiveDir:
  2535. if not os.path.isdir(archiveDir + '/accounts/' + handle):
  2536. os.mkdir(archiveDir + '/accounts/' + handle)
  2537. if not os.path.isdir(archiveDir + '/accounts/' +
  2538. handle + '/inbox'):
  2539. os.mkdir(archiveDir + '/accounts/' +
  2540. handle + '/inbox')
  2541. if not os.path.isdir(archiveDir + '/accounts/' +
  2542. handle + '/outbox'):
  2543. os.mkdir(archiveDir + '/accounts/' +
  2544. handle + '/outbox')
  2545. archiveSubdir = archiveDir + '/accounts/' + \
  2546. handle + '/inbox'
  2547. archivePostsForPerson(httpPrefix, nickname, domain, baseDir,
  2548. 'inbox', archiveSubdir,
  2549. recentPostsCache, maxPostsInBox)
  2550. if archiveDir:
  2551. archiveSubdir = archiveDir + '/accounts/' + \
  2552. handle + '/outbox'
  2553. archivePostsForPerson(httpPrefix, nickname, domain, baseDir,
  2554. 'outbox', archiveSubdir,
  2555. recentPostsCache, maxPostsInBox)
  2556. def archivePostsForPerson(httpPrefix: str, nickname: str, domain: str,
  2557. baseDir: str,
  2558. boxname: str, archiveDir: str,
  2559. recentPostsCache: {},
  2560. maxPostsInBox=32000) -> None:
  2561. """Retain a maximum number of posts within the given box
  2562. Move any others to an archive directory
  2563. """
  2564. if boxname != 'inbox' and boxname != 'outbox':
  2565. return
  2566. if archiveDir:
  2567. if not os.path.isdir(archiveDir):
  2568. os.mkdir(archiveDir)
  2569. boxDir = createPersonDir(nickname, domain, baseDir, boxname)
  2570. postsInBox = os.scandir(boxDir)
  2571. noOfPosts = 0
  2572. for f in postsInBox:
  2573. noOfPosts += 1
  2574. if noOfPosts <= maxPostsInBox:
  2575. print('Checked ' + str(noOfPosts) + ' ' + boxname +
  2576. ' posts for ' + nickname + '@' + domain)
  2577. return
  2578. # remove entries from the index
  2579. handle = nickname + '@' + domain
  2580. indexFilename = baseDir + '/accounts/' + handle + '/' + boxname + '.index'
  2581. if os.path.isfile(indexFilename):
  2582. indexCtr = 0
  2583. # get the existing index entries as a string
  2584. newIndex = ''
  2585. with open(indexFilename, 'r') as indexFile:
  2586. for postId in indexFile:
  2587. newIndex += postId
  2588. indexCtr += 1
  2589. if indexCtr >= maxPostsInBox:
  2590. break
  2591. # save the new index file
  2592. if len(newIndex) > 0:
  2593. indexFile = open(indexFilename, 'w+')
  2594. if indexFile:
  2595. indexFile.write(newIndex)
  2596. indexFile.close()
  2597. postsInBoxDict = {}
  2598. postsCtr = 0
  2599. postsInBox = os.scandir(boxDir)
  2600. for postFilename in postsInBox:
  2601. postFilename = postFilename.name
  2602. if not postFilename.endswith('.json'):
  2603. continue
  2604. # Time of file creation
  2605. fullFilename = os.path.join(boxDir, postFilename)
  2606. if os.path.isfile(fullFilename):
  2607. content = open(fullFilename).read()
  2608. if '"published":' in content:
  2609. publishedStr = content.split('"published":')[1]
  2610. if '"' in publishedStr:
  2611. publishedStr = publishedStr.split('"')[1]
  2612. if publishedStr.endswith('Z'):
  2613. postsInBoxDict[publishedStr] = postFilename
  2614. postsCtr += 1
  2615. noOfPosts = postsCtr
  2616. if noOfPosts <= maxPostsInBox:
  2617. print('Checked ' + str(noOfPosts) + ' ' + boxname +
  2618. ' posts for ' + nickname + '@' + domain)
  2619. return
  2620. # sort the list in ascending order of date
  2621. postsInBoxSorted = \
  2622. OrderedDict(sorted(postsInBoxDict.items(), reverse=False))
  2623. # directory containing cached html posts
  2624. postCacheDir = boxDir.replace('/' + boxname, '/postcache')
  2625. removeCtr = 0
  2626. for publishedStr, postFilename in postsInBoxSorted.items():
  2627. filePath = os.path.join(boxDir, postFilename)
  2628. if not os.path.isfile(filePath):
  2629. continue
  2630. if archiveDir:
  2631. repliesPath = filePath.replace('.json', '.replies')
  2632. archivePath = os.path.join(archiveDir, postFilename)
  2633. os.rename(filePath, archivePath)
  2634. if os.path.isfile(repliesPath):
  2635. os.rename(repliesPath, archivePath)
  2636. else:
  2637. deletePost(baseDir, httpPrefix, nickname, domain,
  2638. filePath, False, recentPostsCache)
  2639. # remove cached html posts
  2640. postCacheFilename = \
  2641. os.path.join(postCacheDir, postFilename).replace('.json', '.html')
  2642. if os.path.isfile(postCacheFilename):
  2643. os.remove(postCacheFilename)
  2644. noOfPosts -= 1
  2645. removeCtr += 1
  2646. if noOfPosts <= maxPostsInBox:
  2647. break
  2648. if archiveDir:
  2649. print('Archived ' + str(removeCtr) + ' ' + boxname +
  2650. ' posts for ' + nickname + '@' + domain)
  2651. else:
  2652. print('Removed ' + str(removeCtr) + ' ' + boxname +
  2653. ' posts for ' + nickname + '@' + domain)
  2654. print(nickname + '@' + domain + ' has ' + str(noOfPosts) +
  2655. ' in ' + boxname)
  2656. def getPublicPostsOfPerson(baseDir: str, nickname: str, domain: str,
  2657. raw: bool, simple: bool, proxyType: str,
  2658. port: int, httpPrefix: str,
  2659. debug: bool, projectVersion: str) -> None:
  2660. """ This is really just for test purposes
  2661. """
  2662. print('Starting new session for getting public posts')
  2663. session = createSession(proxyType)
  2664. if not session:
  2665. return
  2666. personCache = {}
  2667. cachedWebfingers = {}
  2668. federationList = []
  2669. domainFull = domain
  2670. if port:
  2671. if port != 80 and port != 443:
  2672. if ':' not in domain:
  2673. domainFull = domain + ':' + str(port)
  2674. handle = httpPrefix + "://" + domainFull + "/@" + nickname
  2675. wfRequest = \
  2676. webfingerHandle(session, handle, httpPrefix, cachedWebfingers,
  2677. domain, projectVersion)
  2678. if not wfRequest:
  2679. sys.exit()
  2680. if not isinstance(wfRequest, dict):
  2681. print('Webfinger for ' + handle + ' did not return a dict. ' +
  2682. str(wfRequest))
  2683. sys.exit()
  2684. (personUrl, pubKeyId, pubKey,
  2685. personId, shaedInbox,
  2686. capabilityAcquisition,
  2687. avatarUrl, displayName) = getPersonBox(baseDir, session, wfRequest,
  2688. personCache,
  2689. projectVersion, httpPrefix,
  2690. nickname, domain, 'outbox')
  2691. maxMentions = 10
  2692. maxEmoji = 10
  2693. maxAttachments = 5
  2694. getPosts(session, personUrl, 30, maxMentions, maxEmoji,
  2695. maxAttachments, federationList,
  2696. personCache, raw, simple, debug,
  2697. projectVersion, httpPrefix, domain)
  2698. def getPublicPostDomains(baseDir: str, nickname: str, domain: str,
  2699. proxyType: str, port: int, httpPrefix: str,
  2700. debug: bool, projectVersion: str,
  2701. domainList=[]) -> []:
  2702. """ Returns a list of domains referenced within public posts
  2703. """
  2704. session = createSession(proxyType)
  2705. if not session:
  2706. return domainList
  2707. personCache = {}
  2708. cachedWebfingers = {}
  2709. federationList = []
  2710. domainFull = domain
  2711. if port:
  2712. if port != 80 and port != 443:
  2713. if ':' not in domain:
  2714. domainFull = domain + ':' + str(port)
  2715. handle = httpPrefix + "://" + domainFull + "/@" + nickname
  2716. wfRequest = \
  2717. webfingerHandle(session, handle, httpPrefix, cachedWebfingers,
  2718. domain, projectVersion)
  2719. if not wfRequest:
  2720. return domainList
  2721. if not isinstance(wfRequest, dict):
  2722. print('Webfinger for ' + handle + ' did not return a dict. ' +
  2723. str(wfRequest))
  2724. return domainList
  2725. (personUrl, pubKeyId, pubKey,
  2726. personId, shaedInbox,
  2727. capabilityAcquisition,
  2728. avatarUrl, displayName) = getPersonBox(baseDir, session, wfRequest,
  2729. personCache,
  2730. projectVersion, httpPrefix,
  2731. nickname, domain, 'outbox')
  2732. maxMentions = 99
  2733. maxEmoji = 99
  2734. maxAttachments = 5
  2735. postDomains = \
  2736. getPostDomains(session, personUrl, 64, maxMentions, maxEmoji,
  2737. maxAttachments, federationList,
  2738. personCache, debug,
  2739. projectVersion, httpPrefix, domain, domainList)
  2740. postDomains.sort()
  2741. return postDomains
  2742. def sendCapabilitiesUpdate(session, baseDir: str, httpPrefix: str,
  2743. nickname: str, domain: str, port: int,
  2744. followerUrl, updateCaps: [],
  2745. sendThreads: [], postLog: [],
  2746. cachedWebfingers: {}, personCache: {},
  2747. federationList: [], debug: bool,
  2748. projectVersion: str) -> int:
  2749. """When the capabilities for a follower are changed this
  2750. sends out an update. followerUrl is the actor of the follower.
  2751. """
  2752. updateJson = \
  2753. capabilitiesUpdate(baseDir, httpPrefix,
  2754. nickname, domain, port,
  2755. followerUrl, updateCaps)
  2756. if not updateJson:
  2757. return 1
  2758. if debug:
  2759. pprint(updateJson)
  2760. print('DEBUG: sending capabilities update from ' +
  2761. nickname + '@' + domain + ' port ' + str(port) +
  2762. ' to ' + followerUrl)
  2763. clientToServer = False
  2764. followerNickname = getNicknameFromActor(followerUrl)
  2765. if not followerNickname:
  2766. print('WARN: unable to find nickname in ' + followerUrl)
  2767. return 1
  2768. followerDomain, followerPort = getDomainFromActor(followerUrl)
  2769. return sendSignedJson(updateJson, session, baseDir,
  2770. nickname, domain, port,
  2771. followerNickname, followerDomain, followerPort, '',
  2772. httpPrefix, True, clientToServer,
  2773. federationList,
  2774. sendThreads, postLog, cachedWebfingers,
  2775. personCache, debug, projectVersion)
  2776. def populateRepliesJson(baseDir: str, nickname: str, domain: str,
  2777. postRepliesFilename: str, authorized: bool,
  2778. repliesJson: {}) -> None:
  2779. pubStr = 'https://www.w3.org/ns/activitystreams#Public'
  2780. # populate the items list with replies
  2781. repliesBoxes = ('outbox', 'inbox')
  2782. with open(postRepliesFilename, 'r') as repliesFile:
  2783. for messageId in repliesFile:
  2784. replyFound = False
  2785. # examine inbox and outbox
  2786. for boxname in repliesBoxes:
  2787. messageId2 = messageId.replace('\n', '').replace('\r', '')
  2788. searchFilename = \
  2789. baseDir + \
  2790. '/accounts/' + nickname + '@' + \
  2791. domain+'/' + \
  2792. boxname+'/' + \
  2793. messageId2.replace('/', '#') + '.json'
  2794. if os.path.isfile(searchFilename):
  2795. if authorized or \
  2796. pubStr in open(searchFilename).read():
  2797. postJsonObject = loadJson(searchFilename)
  2798. if postJsonObject:
  2799. if postJsonObject['object'].get('cc'):
  2800. pjo = postJsonObject
  2801. if (authorized or
  2802. (pubStr in pjo['object']['to'] or
  2803. pubStr in pjo['object']['cc'])):
  2804. repliesJson['orderedItems'].append(pjo)
  2805. replyFound = True
  2806. else:
  2807. if authorized or \
  2808. pubStr in postJsonObject['object']['to']:
  2809. pjo = postJsonObject
  2810. repliesJson['orderedItems'].append(pjo)
  2811. replyFound = True
  2812. break
  2813. # if not in either inbox or outbox then examine the shared inbox
  2814. if not replyFound:
  2815. messageId2 = messageId.replace('\n', '').replace('\r', '')
  2816. searchFilename = \
  2817. baseDir + \
  2818. '/accounts/inbox@' + \
  2819. domain+'/inbox/' + \
  2820. messageId2.replace('/', '#') + '.json'
  2821. if os.path.isfile(searchFilename):
  2822. if authorized or \
  2823. pubStr in open(searchFilename).read():
  2824. # get the json of the reply and append it to
  2825. # the collection
  2826. postJsonObject = loadJson(searchFilename)
  2827. if postJsonObject:
  2828. if postJsonObject['object'].get('cc'):
  2829. pjo = postJsonObject
  2830. if (authorized or
  2831. (pubStr in pjo['object']['to'] or
  2832. pubStr in pjo['object']['cc'])):
  2833. pjo = postJsonObject
  2834. repliesJson['orderedItems'].append(pjo)
  2835. else:
  2836. if authorized or \
  2837. pubStr in postJsonObject['object']['to']:
  2838. pjo = postJsonObject
  2839. repliesJson['orderedItems'].append(pjo)
  2840. def rejectAnnounce(announceFilename: str):
  2841. """Marks an announce as rejected
  2842. """
  2843. if not os.path.isfile(announceFilename + '.reject'):
  2844. rejectAnnounceFile = open(announceFilename + '.reject', "w+")
  2845. rejectAnnounceFile.write('\n')
  2846. rejectAnnounceFile.close()
  2847. def downloadAnnounce(session, baseDir: str, httpPrefix: str,
  2848. nickname: str, domain: str,
  2849. postJsonObject: {}, projectVersion: str,
  2850. translate: {}) -> {}:
  2851. """Download the post referenced by an announce
  2852. """
  2853. if not postJsonObject.get('object'):
  2854. return None
  2855. if not isinstance(postJsonObject['object'], str):
  2856. return None
  2857. # get the announced post
  2858. announceCacheDir = baseDir + '/cache/announce/' + nickname
  2859. if not os.path.isdir(announceCacheDir):
  2860. os.mkdir(announceCacheDir)
  2861. announceFilename = \
  2862. announceCacheDir + '/' + \
  2863. postJsonObject['object'].replace('/', '#') + '.json'
  2864. if os.path.isfile(announceFilename + '.reject'):
  2865. return None
  2866. if os.path.isfile(announceFilename):
  2867. print('Reading cached Announce content for ' +
  2868. postJsonObject['object'])
  2869. postJsonObject = loadJson(announceFilename)
  2870. if postJsonObject:
  2871. return postJsonObject
  2872. else:
  2873. profileStr = 'https://www.w3.org/ns/activitystreams'
  2874. asHeader = {
  2875. 'Accept': 'application/activity+json; profile="' + profileStr + '"'
  2876. }
  2877. if '/channel/' in postJsonObject['actor']:
  2878. asHeader = {
  2879. 'Accept': 'application/ld+json; profile="' + profileStr + '"'
  2880. }
  2881. actorNickname = getNicknameFromActor(postJsonObject['actor'])
  2882. actorDomain, actorPort = getDomainFromActor(postJsonObject['actor'])
  2883. if not actorDomain:
  2884. print('Announce actor does not contain a ' +
  2885. 'valid domain or port number: ' +
  2886. str(postJsonObject['actor']))
  2887. return None
  2888. if isBlocked(baseDir, nickname, domain, actorNickname, actorDomain):
  2889. print('Announce download blocked actor: ' +
  2890. actorNickname + '@' + actorDomain)
  2891. return None
  2892. objectNickname = getNicknameFromActor(postJsonObject['object'])
  2893. objectDomain, objectPort = getDomainFromActor(postJsonObject['object'])
  2894. if not objectDomain:
  2895. print('Announce object does not contain a ' +
  2896. 'valid domain or port number: ' +
  2897. str(postJsonObject['object']))
  2898. return None
  2899. if isBlocked(baseDir, nickname, domain, objectNickname, objectDomain):
  2900. if objectNickname and objectDomain:
  2901. print('Announce download blocked object: ' +
  2902. objectNickname + '@' + objectDomain)
  2903. else:
  2904. print('Announce download blocked object: ' +
  2905. str(postJsonObject['object']))
  2906. return None
  2907. print('Downloading Announce content for ' + postJsonObject['object'])
  2908. announcedJson = \
  2909. getJson(session, postJsonObject['object'], asHeader,
  2910. None, projectVersion, httpPrefix, domain)
  2911. if not announcedJson:
  2912. return None
  2913. if not isinstance(announcedJson, dict):
  2914. print('WARN: announce json is not a dict - ' +
  2915. postJsonObject['object'])
  2916. rejectAnnounce(announceFilename)
  2917. return None
  2918. if not announcedJson.get('id'):
  2919. rejectAnnounce(announceFilename)
  2920. return None
  2921. if '/statuses/' not in announcedJson['id']:
  2922. rejectAnnounce(announceFilename)
  2923. return None
  2924. if '/users/' not in announcedJson['id'] and \
  2925. '/channel/' not in announcedJson['id'] and \
  2926. '/profile/' not in announcedJson['id']:
  2927. rejectAnnounce(announceFilename)
  2928. return None
  2929. if not announcedJson.get('type'):
  2930. rejectAnnounce(announceFilename)
  2931. # pprint(announcedJson)
  2932. return None
  2933. if announcedJson['type'] != 'Note' and \
  2934. announcedJson['type'] != 'Article':
  2935. rejectAnnounce(announceFilename)
  2936. # pprint(announcedJson)
  2937. return None
  2938. if not announcedJson.get('content'):
  2939. rejectAnnounce(announceFilename)
  2940. return None
  2941. if isFiltered(baseDir, nickname, domain, announcedJson['content']):
  2942. rejectAnnounce(announceFilename)
  2943. return None
  2944. # remove any long words
  2945. announcedJson['content'] = \
  2946. removeLongWords(announcedJson['content'], 40, [])
  2947. # remove text formatting, such as bold/italics
  2948. announcedJson['content'] = \
  2949. removeTextFormatting(announcedJson['content'])
  2950. # wrap in create to be consistent with other posts
  2951. announcedJson = \
  2952. outboxMessageCreateWrap(httpPrefix,
  2953. actorNickname, actorDomain, actorPort,
  2954. announcedJson)
  2955. if announcedJson['type'] != 'Create':
  2956. rejectAnnounce(announceFilename)
  2957. # pprint(announcedJson)
  2958. return None
  2959. # labelAccusatoryPost(postJsonObject, translate)
  2960. # set the id to the original status
  2961. announcedJson['id'] = postJsonObject['object']
  2962. announcedJson['object']['id'] = postJsonObject['object']
  2963. # check that the repeat isn't for a blocked account
  2964. attributedNickname = \
  2965. getNicknameFromActor(announcedJson['object']['id'])
  2966. attributedDomain, attributedPort = \
  2967. getDomainFromActor(announcedJson['object']['id'])
  2968. if attributedNickname and attributedDomain:
  2969. if attributedPort:
  2970. if attributedPort != 80 and attributedPort != 443:
  2971. attributedDomain = \
  2972. attributedDomain + ':' + str(attributedPort)
  2973. if isBlocked(baseDir, nickname, domain,
  2974. attributedNickname, attributedDomain):
  2975. rejectAnnounce(announceFilename)
  2976. return None
  2977. postJsonObject = announcedJson
  2978. replaceYouTube(postJsonObject)
  2979. if saveJson(postJsonObject, announceFilename):
  2980. return postJsonObject
  2981. return None
  2982. def mutePost(baseDir: str, nickname: str, domain: str, postId: str,
  2983. recentPostsCache: {}) -> None:
  2984. """ Mutes the given post
  2985. """
  2986. postFilename = locatePost(baseDir, nickname, domain, postId)
  2987. if not postFilename:
  2988. return
  2989. postJsonObject = loadJson(postFilename)
  2990. if not postJsonObject:
  2991. return
  2992. print('MUTE: ' + postFilename)
  2993. muteFile = open(postFilename + '.muted', "w")
  2994. if muteFile:
  2995. muteFile.write('\n')
  2996. muteFile.close()
  2997. # remove cached posts so that the muted version gets created
  2998. cachedPostFilename = \
  2999. getCachedPostFilename(baseDir, nickname, domain, postJsonObject)
  3000. if cachedPostFilename:
  3001. if os.path.isfile(cachedPostFilename):
  3002. os.remove(cachedPostFilename)
  3003. # if the post is in the recent posts cache then mark it as muted
  3004. if recentPostsCache.get('index'):
  3005. postId = \
  3006. postJsonObject['id'].replace('/activity', '').replace('/', '#')
  3007. if postId in recentPostsCache['index']:
  3008. print('MUTE: ' + postId + ' is in recent posts cache')
  3009. if recentPostsCache['json'].get(postId):
  3010. postJsonObject['muted'] = True
  3011. recentPostsCache['json'][postId] = json.dumps(postJsonObject)
  3012. print('MUTE: ' + postId +
  3013. ' marked as muted in recent posts cache')
  3014. def unmutePost(baseDir: str, nickname: str, domain: str, postId: str,
  3015. recentPostsCache: {}) -> None:
  3016. """ Unmutes the given post
  3017. """
  3018. postFilename = locatePost(baseDir, nickname, domain, postId)
  3019. if not postFilename:
  3020. return
  3021. postJsonObject = loadJson(postFilename)
  3022. if not postJsonObject:
  3023. return
  3024. print('UNMUTE: ' + postFilename)
  3025. muteFilename = postFilename + '.muted'
  3026. if os.path.isfile(muteFilename):
  3027. os.remove(muteFilename)
  3028. # remove cached posts so that it gets recreated
  3029. cachedPostFilename = \
  3030. getCachedPostFilename(baseDir, nickname, domain, postJsonObject)
  3031. if cachedPostFilename:
  3032. if os.path.isfile(cachedPostFilename):
  3033. os.remove(cachedPostFilename)
  3034. removePostFromCache(postJsonObject, recentPostsCache)
  3035. def sendBlockViaServer(baseDir: str, session,
  3036. fromNickname: str, password: str,
  3037. fromDomain: str, fromPort: int,
  3038. httpPrefix: str, blockedUrl: str,
  3039. cachedWebfingers: {}, personCache: {},
  3040. debug: bool, projectVersion: str) -> {}:
  3041. """Creates a block via c2s
  3042. """
  3043. if not session:
  3044. print('WARN: No session for sendBlockViaServer')
  3045. return 6
  3046. fromDomainFull = fromDomain
  3047. if fromPort:
  3048. if fromPort != 80 and fromPort != 443:
  3049. if ':' not in fromDomain:
  3050. fromDomainFull = fromDomain + ':' + str(fromPort)
  3051. toUrl = 'https://www.w3.org/ns/activitystreams#Public'
  3052. ccUrl = httpPrefix + '://' + fromDomainFull + '/users/' + \
  3053. fromNickname + '/followers'
  3054. blockActor = httpPrefix + '://' + fromDomainFull + '/users/' + fromNickname
  3055. newBlockJson = {
  3056. "@context": "https://www.w3.org/ns/activitystreams",
  3057. 'type': 'Block',
  3058. 'actor': blockActor,
  3059. 'object': blockedUrl,
  3060. 'to': [toUrl],
  3061. 'cc': [ccUrl]
  3062. }
  3063. handle = httpPrefix + '://' + fromDomainFull + '/@' + fromNickname
  3064. # lookup the inbox for the To handle
  3065. wfRequest = webfingerHandle(session, handle, httpPrefix,
  3066. cachedWebfingers,
  3067. fromDomain, projectVersion)
  3068. if not wfRequest:
  3069. if debug:
  3070. print('DEBUG: announce webfinger failed for ' + handle)
  3071. return 1
  3072. if not isinstance(wfRequest, dict):
  3073. print('WARN: Webfinger for ' + handle + ' did not return a dict. ' +
  3074. str(wfRequest))
  3075. return 1
  3076. postToBox = 'outbox'
  3077. # get the actor inbox for the To handle
  3078. (inboxUrl, pubKeyId, pubKey,
  3079. fromPersonId, sharedInbox,
  3080. capabilityAcquisition, avatarUrl,
  3081. displayName) = getPersonBox(baseDir, session, wfRequest,
  3082. personCache,
  3083. projectVersion, httpPrefix, fromNickname,
  3084. fromDomain, postToBox)
  3085. if not inboxUrl:
  3086. if debug:
  3087. print('DEBUG: No ' + postToBox + ' was found for ' + handle)
  3088. return 3
  3089. if not fromPersonId:
  3090. if debug:
  3091. print('DEBUG: No actor was found for ' + handle)
  3092. return 4
  3093. authHeader = createBasicAuthHeader(fromNickname, password)
  3094. headers = {
  3095. 'host': fromDomain,
  3096. 'Content-type': 'application/json',
  3097. 'Authorization': authHeader
  3098. }
  3099. postResult = postJson(session, newBlockJson, [], inboxUrl,
  3100. headers, "inbox:write")
  3101. if not postResult:
  3102. print('WARN: Unable to post block')
  3103. if debug:
  3104. print('DEBUG: c2s POST block success')
  3105. return newBlockJson
  3106. def sendUndoBlockViaServer(baseDir: str, session,
  3107. fromNickname: str, password: str,
  3108. fromDomain: str, fromPort: int,
  3109. httpPrefix: str, blockedUrl: str,
  3110. cachedWebfingers: {}, personCache: {},
  3111. debug: bool, projectVersion: str) -> {}:
  3112. """Creates a block via c2s
  3113. """
  3114. if not session:
  3115. print('WARN: No session for sendBlockViaServer')
  3116. return 6
  3117. fromDomainFull = fromDomain
  3118. if fromPort:
  3119. if fromPort != 80 and fromPort != 443:
  3120. if ':' not in fromDomain:
  3121. fromDomainFull = fromDomain + ':' + str(fromPort)
  3122. toUrl = 'https://www.w3.org/ns/activitystreams#Public'
  3123. ccUrl = httpPrefix + '://' + fromDomainFull + '/users/' + \
  3124. fromNickname + '/followers'
  3125. blockActor = httpPrefix + '://' + fromDomainFull + '/users/' + fromNickname
  3126. newBlockJson = {
  3127. "@context": "https://www.w3.org/ns/activitystreams",
  3128. 'type': 'Undo',
  3129. 'actor': blockActor,
  3130. 'object': {
  3131. 'type': 'Block',
  3132. 'actor': blockActor,
  3133. 'object': blockedUrl,
  3134. 'to': [toUrl],
  3135. 'cc': [ccUrl]
  3136. }
  3137. }
  3138. handle = httpPrefix + '://' + fromDomainFull + '/@' + fromNickname
  3139. # lookup the inbox for the To handle
  3140. wfRequest = webfingerHandle(session, handle, httpPrefix,
  3141. cachedWebfingers,
  3142. fromDomain, projectVersion)
  3143. if not wfRequest:
  3144. if debug:
  3145. print('DEBUG: announce webfinger failed for ' + handle)
  3146. return 1
  3147. if not isinstance(wfRequest, dict):
  3148. print('WARN: Webfinger for ' + handle + ' did not return a dict. ' +
  3149. str(wfRequest))
  3150. return 1
  3151. postToBox = 'outbox'
  3152. # get the actor inbox for the To handle
  3153. (inboxUrl, pubKeyId, pubKey,
  3154. fromPersonId, sharedInbox,
  3155. capabilityAcquisition, avatarUrl,
  3156. displayName) = getPersonBox(baseDir, session, wfRequest, personCache,
  3157. projectVersion, httpPrefix, fromNickname,
  3158. fromDomain, postToBox)
  3159. if not inboxUrl:
  3160. if debug:
  3161. print('DEBUG: No ' + postToBox + ' was found for ' + handle)
  3162. return 3
  3163. if not fromPersonId:
  3164. if debug:
  3165. print('DEBUG: No actor was found for ' + handle)
  3166. return 4
  3167. authHeader = createBasicAuthHeader(fromNickname, password)
  3168. headers = {
  3169. 'host': fromDomain,
  3170. 'Content-type': 'application/json',
  3171. 'Authorization': authHeader
  3172. }
  3173. postResult = postJson(session, newBlockJson, [], inboxUrl,
  3174. headers, "inbox:write")
  3175. if not postResult:
  3176. print('WARN: Unable to post block')
  3177. if debug:
  3178. print('DEBUG: c2s POST block success')
  3179. return newBlockJson