inbox.py 113 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957958959960961962963964965966967968969970971972973974975976977978979980981982983984985986987988989990991992993994995996997998999100010011002100310041005100610071008100910101011101210131014101510161017101810191020102110221023102410251026102710281029103010311032103310341035103610371038103910401041104210431044104510461047104810491050105110521053105410551056105710581059106010611062106310641065106610671068106910701071107210731074107510761077107810791080108110821083108410851086108710881089109010911092109310941095109610971098109911001101110211031104110511061107110811091110111111121113111411151116111711181119112011211122112311241125112611271128112911301131113211331134113511361137113811391140114111421143114411451146114711481149115011511152115311541155115611571158115911601161116211631164116511661167116811691170117111721173117411751176117711781179118011811182118311841185118611871188118911901191119211931194119511961197119811991200120112021203120412051206120712081209121012111212121312141215121612171218121912201221122212231224122512261227122812291230123112321233123412351236123712381239124012411242124312441245124612471248124912501251125212531254125512561257125812591260126112621263126412651266126712681269127012711272127312741275127612771278127912801281128212831284128512861287128812891290129112921293129412951296129712981299130013011302130313041305130613071308130913101311131213131314131513161317131813191320132113221323132413251326132713281329133013311332133313341335133613371338133913401341134213431344134513461347134813491350135113521353135413551356135713581359136013611362136313641365136613671368136913701371137213731374137513761377137813791380138113821383138413851386138713881389139013911392139313941395139613971398139914001401140214031404140514061407140814091410141114121413141414151416141714181419142014211422142314241425142614271428142914301431143214331434143514361437143814391440144114421443144414451446144714481449145014511452145314541455145614571458145914601461146214631464146514661467146814691470147114721473147414751476147714781479148014811482148314841485148614871488148914901491149214931494149514961497149814991500150115021503150415051506150715081509151015111512151315141515151615171518151915201521152215231524152515261527152815291530153115321533153415351536153715381539154015411542154315441545154615471548154915501551155215531554155515561557155815591560156115621563156415651566156715681569157015711572157315741575157615771578157915801581158215831584158515861587158815891590159115921593159415951596159715981599160016011602160316041605160616071608160916101611161216131614161516161617161816191620162116221623162416251626162716281629163016311632163316341635163616371638163916401641164216431644164516461647164816491650165116521653165416551656165716581659166016611662166316641665166616671668166916701671167216731674167516761677167816791680168116821683168416851686168716881689169016911692169316941695169616971698169917001701170217031704170517061707170817091710171117121713171417151716171717181719172017211722172317241725172617271728172917301731173217331734173517361737173817391740174117421743174417451746174717481749175017511752175317541755175617571758175917601761176217631764176517661767176817691770177117721773177417751776177717781779178017811782178317841785178617871788178917901791179217931794179517961797179817991800180118021803180418051806180718081809181018111812181318141815181618171818181918201821182218231824182518261827182818291830183118321833183418351836183718381839184018411842184318441845184618471848184918501851185218531854185518561857185818591860186118621863186418651866186718681869187018711872187318741875187618771878187918801881188218831884188518861887188818891890189118921893189418951896189718981899190019011902190319041905190619071908190919101911191219131914191519161917191819191920192119221923192419251926192719281929193019311932193319341935193619371938193919401941194219431944194519461947194819491950195119521953195419551956195719581959196019611962196319641965196619671968196919701971197219731974197519761977197819791980198119821983198419851986198719881989199019911992199319941995199619971998199920002001200220032004200520062007200820092010201120122013201420152016201720182019202020212022202320242025202620272028202920302031203220332034203520362037203820392040204120422043204420452046204720482049205020512052205320542055205620572058205920602061206220632064206520662067206820692070207120722073207420752076207720782079208020812082208320842085208620872088208920902091209220932094209520962097209820992100210121022103210421052106210721082109211021112112211321142115211621172118211921202121212221232124212521262127212821292130213121322133213421352136213721382139214021412142214321442145214621472148214921502151215221532154215521562157215821592160216121622163216421652166216721682169217021712172217321742175217621772178217921802181218221832184218521862187218821892190219121922193219421952196219721982199220022012202220322042205220622072208220922102211221222132214221522162217221822192220222122222223222422252226222722282229223022312232223322342235223622372238223922402241224222432244224522462247224822492250225122522253225422552256225722582259226022612262226322642265226622672268226922702271227222732274227522762277227822792280228122822283228422852286228722882289229022912292229322942295229622972298229923002301230223032304230523062307230823092310231123122313231423152316231723182319232023212322232323242325232623272328232923302331233223332334233523362337233823392340234123422343234423452346234723482349235023512352235323542355235623572358235923602361236223632364236523662367236823692370237123722373237423752376237723782379238023812382238323842385238623872388238923902391239223932394239523962397239823992400240124022403240424052406240724082409241024112412241324142415241624172418241924202421242224232424242524262427242824292430243124322433243424352436243724382439244024412442244324442445244624472448244924502451245224532454245524562457245824592460246124622463246424652466246724682469247024712472247324742475247624772478247924802481248224832484248524862487248824892490249124922493249424952496249724982499250025012502250325042505250625072508250925102511251225132514251525162517251825192520252125222523252425252526252725282529253025312532253325342535253625372538253925402541254225432544254525462547254825492550255125522553255425552556255725582559256025612562256325642565256625672568256925702571257225732574257525762577257825792580258125822583258425852586258725882589259025912592259325942595259625972598259926002601260226032604260526062607260826092610261126122613261426152616261726182619262026212622262326242625262626272628262926302631263226332634263526362637263826392640264126422643264426452646264726482649265026512652265326542655265626572658265926602661266226632664266526662667266826692670267126722673267426752676267726782679268026812682268326842685268626872688268926902691269226932694269526962697269826992700270127022703270427052706270727082709271027112712271327142715271627172718271927202721272227232724272527262727272827292730273127322733273427352736273727382739274027412742274327442745274627472748274927502751275227532754275527562757275827592760276127622763276427652766276727682769277027712772277327742775277627772778277927802781278227832784278527862787278827892790279127922793279427952796279727982799280028012802280328042805280628072808280928102811281228132814281528162817281828192820282128222823282428252826282728282829283028312832283328342835283628372838283928402841284228432844284528462847284828492850285128522853285428552856285728582859
  1. __filename__ = "inbox.py"
  2. __author__ = "Bob Mottram"
  3. __license__ = "AGPL3+"
  4. __version__ = "1.1.0"
  5. __maintainer__ = "Bob Mottram"
  6. __email__ = "bob@freedombone.net"
  7. __status__ = "Production"
  8. import json
  9. import os
  10. import datetime
  11. import time
  12. from utils import getProtocolPrefixes
  13. from utils import isBlogPost
  14. from utils import removeAvatarFromCache
  15. from utils import isPublicPost
  16. from utils import getCachedPostFilename
  17. from utils import removePostFromCache
  18. from utils import urlPermitted
  19. from utils import createInboxQueueDir
  20. from utils import getStatusNumber
  21. from utils import getDomainFromActor
  22. from utils import getNicknameFromActor
  23. from utils import locatePost
  24. from utils import deletePost
  25. from utils import removeModerationPostFromIndex
  26. from utils import loadJson
  27. from utils import saveJson
  28. from httpsig import verifyPostHeaders
  29. from session import createSession
  30. from session import getJson
  31. from follow import receiveFollowRequest
  32. from follow import getFollowersOfActor
  33. from follow import unfollowerOfPerson
  34. from pprint import pprint
  35. from cache import getPersonFromCache
  36. from cache import storePersonInCache
  37. from acceptreject import receiveAcceptReject
  38. from capabilities import getOcapFilename
  39. from capabilities import CapablePost
  40. from capabilities import capabilitiesReceiveUpdate
  41. from like import updateLikesCollection
  42. from like import undoLikesCollectionEntry
  43. from bookmarks import updateBookmarksCollection
  44. from bookmarks import undoBookmarksCollectionEntry
  45. from blocking import isBlocked
  46. from blocking import isBlockedDomain
  47. from filters import isFiltered
  48. from announce import updateAnnounceCollection
  49. from announce import undoAnnounceCollectionEntry
  50. from httpsig import messageContentDigest
  51. from posts import downloadAnnounce
  52. from posts import isDM
  53. from posts import isReply
  54. from posts import isImageMedia
  55. from posts import sendSignedJson
  56. from posts import sendToFollowersThread
  57. from webinterface import individualPostAsHtml
  58. from webinterface import getIconsDir
  59. from webinterface import removeOldHashtags
  60. from question import questionUpdateVotes
  61. from media import replaceYouTube
  62. from git import isGitPatch
  63. from git import receiveGitPatch
  64. from followingCalendar import receivingCalendarEvents
  65. from content import dangerousMarkup
  66. from happening import saveEvent
  67. def storeHashTags(baseDir: str, nickname: str, postJsonObject: {}) -> None:
  68. """Extracts hashtags from an incoming post and updates the
  69. relevant tags files.
  70. """
  71. if not isPublicPost(postJsonObject):
  72. return
  73. if not postJsonObject.get('object'):
  74. return
  75. if not isinstance(postJsonObject['object'], dict):
  76. return
  77. if not postJsonObject['object'].get('tag'):
  78. return
  79. if not postJsonObject.get('id'):
  80. return
  81. if not isinstance(postJsonObject['object']['tag'], list):
  82. return
  83. tagsDir = baseDir+'/tags'
  84. for tag in postJsonObject['object']['tag']:
  85. if not tag.get('type'):
  86. continue
  87. if tag['type'] != 'Hashtag':
  88. continue
  89. if not tag.get('name'):
  90. continue
  91. tagName = tag['name'].replace('#', '').strip()
  92. tagsFilename = tagsDir + '/' + tagName + '.txt'
  93. postUrl = postJsonObject['id'].replace('/activity', '')
  94. postUrl = postUrl.replace('/', '#')
  95. daysDiff = datetime.datetime.utcnow() - datetime.datetime(1970, 1, 1)
  96. daysSinceEpoch = daysDiff.days
  97. tagline = str(daysSinceEpoch) + ' ' + nickname + ' ' + postUrl + '\n'
  98. if not os.path.isfile(tagsFilename):
  99. tagsFile = open(tagsFilename, "w+")
  100. if tagsFile:
  101. tagsFile.write(tagline)
  102. tagsFile.close()
  103. else:
  104. if postUrl not in open(tagsFilename).read():
  105. try:
  106. with open(tagsFilename, 'r+') as tagsFile:
  107. content = tagsFile.read()
  108. tagsFile.seek(0, 0)
  109. tagsFile.write(tagline + content)
  110. except Exception as e:
  111. print('WARN: Failed to write entry to tags file ' +
  112. tagsFilename + ' ' + str(e))
  113. removeOldHashtags(baseDir, 3)
  114. def inboxStorePostToHtmlCache(recentPostsCache: {}, maxRecentPosts: int,
  115. translate: {},
  116. baseDir: str, httpPrefix: str,
  117. session, cachedWebfingers: {}, personCache: {},
  118. nickname: str, domain: str, port: int,
  119. postJsonObject: {},
  120. allowDeletion: bool) -> None:
  121. """Converts the json post into html and stores it in a cache
  122. This enables the post to be quickly displayed later
  123. """
  124. pageNumber = -999
  125. avatarUrl = None
  126. boxName = 'inbox'
  127. individualPostAsHtml(recentPostsCache, maxRecentPosts,
  128. getIconsDir(baseDir), translate, pageNumber,
  129. baseDir, session, cachedWebfingers, personCache,
  130. nickname, domain, port, postJsonObject,
  131. avatarUrl, True, allowDeletion,
  132. httpPrefix, __version__, boxName,
  133. not isDM(postJsonObject),
  134. True, True, False, True)
  135. def validInbox(baseDir: str, nickname: str, domain: str) -> bool:
  136. """Checks whether files were correctly saved to the inbox
  137. """
  138. if ':' in domain:
  139. domain = domain.split(':')[0]
  140. inboxDir = baseDir+'/accounts/' + nickname + '@' + domain + '/inbox'
  141. if not os.path.isdir(inboxDir):
  142. return True
  143. for subdir, dirs, files in os.walk(inboxDir):
  144. for f in files:
  145. filename = os.path.join(subdir, f)
  146. if not os.path.isfile(filename):
  147. print('filename: ' + filename)
  148. return False
  149. if 'postNickname' in open(filename).read():
  150. print('queue file incorrectly saved to ' + filename)
  151. return False
  152. return True
  153. def validInboxFilenames(baseDir: str, nickname: str, domain: str,
  154. expectedDomain: str, expectedPort: int) -> bool:
  155. """Used by unit tests to check that the port number gets appended to
  156. domain names within saved post filenames
  157. """
  158. if ':' in domain:
  159. domain = domain.split(':')[0]
  160. inboxDir = baseDir + '/accounts/' + nickname + '@' + domain + '/inbox'
  161. if not os.path.isdir(inboxDir):
  162. return True
  163. expectedStr = expectedDomain + ':' + str(expectedPort)
  164. for subdir, dirs, files in os.walk(inboxDir):
  165. for f in files:
  166. filename = os.path.join(subdir, f)
  167. if not os.path.isfile(filename):
  168. print('filename: ' + filename)
  169. return False
  170. if expectedStr not in filename:
  171. print('Expected: ' + expectedStr)
  172. print('Invalid filename: ' + filename)
  173. return False
  174. return True
  175. def getPersonPubKey(baseDir: str, session, personUrl: str,
  176. personCache: {}, debug: bool,
  177. projectVersion: str, httpPrefix: str,
  178. domain: str, onionDomain: str) -> str:
  179. if not personUrl:
  180. return None
  181. personUrl = personUrl.replace('#main-key', '')
  182. if personUrl.endswith('/users/inbox'):
  183. if debug:
  184. print('DEBUG: Obtaining public key for shared inbox')
  185. personUrl = personUrl.replace('/users/inbox', '/inbox')
  186. personJson = getPersonFromCache(baseDir, personUrl, personCache)
  187. if not personJson:
  188. if debug:
  189. print('DEBUG: Obtaining public key for ' + personUrl)
  190. personDomain = domain
  191. if onionDomain:
  192. if '.onion/' in personUrl:
  193. personDomain = onionDomain
  194. profileStr = 'https://www.w3.org/ns/activitystreams'
  195. asHeader = {
  196. 'Accept': 'application/activity+json; profile="' + profileStr + '"'
  197. }
  198. personJson = \
  199. getJson(session, personUrl, asHeader, None, projectVersion,
  200. httpPrefix, personDomain)
  201. if not personJson:
  202. return None
  203. pubKey = None
  204. if personJson.get('publicKey'):
  205. if personJson['publicKey'].get('publicKeyPem'):
  206. pubKey = personJson['publicKey']['publicKeyPem']
  207. else:
  208. if personJson.get('publicKeyPem'):
  209. pubKey = personJson['publicKeyPem']
  210. if not pubKey:
  211. if debug:
  212. print('DEBUG: Public key not found for ' + personUrl)
  213. storePersonInCache(baseDir, personUrl, personJson, personCache)
  214. return pubKey
  215. def inboxMessageHasParams(messageJson: {}) -> bool:
  216. """Checks whether an incoming message contains expected parameters
  217. """
  218. expectedParams = ['type', 'actor', 'object']
  219. for param in expectedParams:
  220. if not messageJson.get(param):
  221. return False
  222. if not messageJson.get('to'):
  223. allowedWithoutToParam = ['Like', 'Follow', 'Request',
  224. 'Accept', 'Capability', 'Undo']
  225. if messageJson['type'] not in allowedWithoutToParam:
  226. return False
  227. return True
  228. def inboxPermittedMessage(domain: str, messageJson: {},
  229. federationList: []) -> bool:
  230. """ check that we are receiving from a permitted domain
  231. """
  232. if not messageJson.get('actor'):
  233. return False
  234. actor = messageJson['actor']
  235. # always allow the local domain
  236. if domain in actor:
  237. return True
  238. if not urlPermitted(actor, federationList, "inbox:write"):
  239. return False
  240. alwaysAllowedTypes = ('Follow', 'Like', 'Delete', 'Announce')
  241. if messageJson['type'] not in alwaysAllowedTypes:
  242. if not messageJson.get('object'):
  243. return True
  244. if not isinstance(messageJson['object'], dict):
  245. return False
  246. if messageJson['object'].get('inReplyTo'):
  247. inReplyTo = messageJson['object']['inReplyTo']
  248. if not urlPermitted(inReplyTo, federationList, "inbox:write"):
  249. return False
  250. return True
  251. def validPublishedDate(published: str) -> bool:
  252. currTime = datetime.datetime.utcnow()
  253. pubDate = datetime.datetime.strptime(published, "%Y-%m-%dT%H:%M:%SZ")
  254. daysSincePublished = (currTime - pubDate).days
  255. if daysSincePublished > 30:
  256. return False
  257. return True
  258. def savePostToInboxQueue(baseDir: str, httpPrefix: str,
  259. nickname: str, domain: str,
  260. postJsonObject: {},
  261. messageBytes: str,
  262. httpHeaders: {},
  263. postPath: str, debug: bool) -> str:
  264. """Saves the give json to the inbox queue for the person
  265. keyId specifies the actor sending the post
  266. """
  267. if len(messageBytes) > 10240:
  268. print('WARN: inbox message too long ' +
  269. str(len(messageBytes)) + ' bytes')
  270. return None
  271. originalDomain = domain
  272. if ':' in domain:
  273. domain = domain.split(':')[0]
  274. # block at the ealiest stage possible, which means the data
  275. # isn't written to file
  276. postNickname = None
  277. postDomain = None
  278. actor = None
  279. if postJsonObject.get('actor'):
  280. actor = postJsonObject['actor']
  281. postNickname = getNicknameFromActor(postJsonObject['actor'])
  282. if not postNickname:
  283. print('No post Nickname in actor ' + postJsonObject['actor'])
  284. return None
  285. postDomain, postPort = getDomainFromActor(postJsonObject['actor'])
  286. if not postDomain:
  287. if debug:
  288. pprint(postJsonObject)
  289. print('No post Domain in actor')
  290. return None
  291. if isBlocked(baseDir, nickname, domain, postNickname, postDomain):
  292. if debug:
  293. print('DEBUG: post from ' + postNickname + ' blocked')
  294. return None
  295. if postPort:
  296. if postPort != 80 and postPort != 443:
  297. if ':' not in postDomain:
  298. postDomain = postDomain + ':' + str(postPort)
  299. if postJsonObject.get('object'):
  300. if isinstance(postJsonObject['object'], dict):
  301. if postJsonObject['object'].get('inReplyTo'):
  302. if isinstance(postJsonObject['object']['inReplyTo'], str):
  303. inReplyTo = \
  304. postJsonObject['object']['inReplyTo']
  305. replyDomain, replyPort = \
  306. getDomainFromActor(inReplyTo)
  307. if isBlockedDomain(baseDir, replyDomain):
  308. print('WARN: post contains reply from ' +
  309. str(actor) +
  310. ' to a blocked domain: ' + replyDomain)
  311. return None
  312. else:
  313. replyNickname = \
  314. getNicknameFromActor(inReplyTo)
  315. if replyNickname and replyDomain:
  316. if isBlocked(baseDir, nickname, domain,
  317. replyNickname, replyDomain):
  318. print('WARN: post contains reply from ' +
  319. str(actor) +
  320. ' to a blocked account: ' +
  321. replyNickname + '@' + replyDomain)
  322. return None
  323. if postJsonObject['object'].get('content'):
  324. if isinstance(postJsonObject['object']['content'], str):
  325. if isFiltered(baseDir, nickname, domain,
  326. postJsonObject['object']['content']):
  327. print('WARN: post was filtered out due to content')
  328. return None
  329. originalPostId = None
  330. if postJsonObject.get('id'):
  331. originalPostId = \
  332. postJsonObject['id'].replace('/activity', '').replace('/undo', '')
  333. currTime = datetime.datetime.utcnow()
  334. postId = None
  335. if postJsonObject.get('id'):
  336. postId = postJsonObject['id'].replace('/activity', '')
  337. postId = postId.replace('/undo', '')
  338. published = currTime.strftime("%Y-%m-%dT%H:%M:%SZ")
  339. if not postId:
  340. statusNumber, published = getStatusNumber()
  341. if actor:
  342. postId = actor + '/statuses/' + statusNumber
  343. else:
  344. postId = httpPrefix + '://' + originalDomain + \
  345. '/users/' + nickname + '/statuses/' + statusNumber
  346. # NOTE: don't change postJsonObject['id'] before signature check
  347. inboxQueueDir = createInboxQueueDir(nickname, domain, baseDir)
  348. handle = nickname + '@' + domain
  349. destination = baseDir + '/accounts/' + \
  350. handle + '/inbox/' + postId.replace('/', '#') + '.json'
  351. filename = inboxQueueDir + '/' + postId.replace('/', '#') + '.json'
  352. sharedInboxItem = False
  353. if nickname == 'inbox':
  354. nickname = originalDomain
  355. sharedInboxItem = True
  356. digestStartTime = time.time()
  357. digest = messageContentDigest(messageBytes)
  358. timeDiffStr = str(int((time.time() - digestStartTime) * 1000))
  359. if debug:
  360. while len(timeDiffStr) < 6:
  361. timeDiffStr = '0' + timeDiffStr
  362. print('DIGEST|' + timeDiffStr + '|' + filename)
  363. newQueueItem = {
  364. 'originalId': originalPostId,
  365. 'id': postId,
  366. 'actor': actor,
  367. 'nickname': nickname,
  368. 'domain': domain,
  369. 'postNickname': postNickname,
  370. 'postDomain': postDomain,
  371. 'sharedInbox': sharedInboxItem,
  372. 'published': published,
  373. 'httpHeaders': httpHeaders,
  374. 'path': postPath,
  375. 'post': postJsonObject,
  376. 'digest': digest,
  377. 'filename': filename,
  378. 'destination': destination
  379. }
  380. if debug:
  381. print('Inbox queue item created')
  382. saveJson(newQueueItem, filename)
  383. return filename
  384. def inboxCheckCapabilities(baseDir: str, nickname: str, domain: str,
  385. actor: str, queueFilename: str, queue: [],
  386. queueJson: {}, capabilityId: str,
  387. debug: bool) -> bool:
  388. if nickname == 'inbox':
  389. return True
  390. ocapFilename = \
  391. getOcapFilename(baseDir,
  392. queueJson['nickname'], queueJson['domain'],
  393. actor, 'accept')
  394. if not ocapFilename:
  395. return False
  396. if not os.path.isfile(ocapFilename):
  397. if debug:
  398. print('DEBUG: capabilities for ' +
  399. actor + ' do not exist')
  400. if os.path.isfile(queueFilename):
  401. os.remove(queueFilename)
  402. if len(queue) > 0:
  403. queue.pop(0)
  404. return False
  405. oc = loadJson(ocapFilename)
  406. if not oc:
  407. return False
  408. if not oc.get('id'):
  409. if debug:
  410. print('DEBUG: capabilities for ' + actor + ' do not contain an id')
  411. if os.path.isfile(queueFilename):
  412. os.remove(queueFilename)
  413. if len(queue) > 0:
  414. queue.pop(0)
  415. return False
  416. if oc['id'] != capabilityId:
  417. if debug:
  418. print('DEBUG: capability id mismatch')
  419. if os.path.isfile(queueFilename):
  420. os.remove(queueFilename)
  421. if len(queue) > 0:
  422. queue.pop(0)
  423. return False
  424. if not oc.get('capability'):
  425. if debug:
  426. print('DEBUG: missing capability list')
  427. if os.path.isfile(queueFilename):
  428. os.remove(queueFilename)
  429. if len(queue) > 0:
  430. queue.pop(0)
  431. return False
  432. if not CapablePost(queueJson['post'], oc['capability'], debug):
  433. if debug:
  434. print('DEBUG: insufficient capabilities to write to inbox from ' +
  435. actor)
  436. if os.path.isfile(queueFilename):
  437. os.remove(queueFilename)
  438. if len(queue) > 0:
  439. queue.pop(0)
  440. return False
  441. if debug:
  442. print('DEBUG: object capabilities check success')
  443. return True
  444. def inboxPostRecipientsAdd(baseDir: str, httpPrefix: str, toList: [],
  445. recipientsDict: {},
  446. domainMatch: str, domain: str,
  447. actor: str, debug: bool) -> bool:
  448. """Given a list of post recipients (toList) from 'to' or 'cc' parameters
  449. populate a recipientsDict with the handle and capabilities id for each
  450. """
  451. followerRecipients = False
  452. for recipient in toList:
  453. if not recipient:
  454. continue
  455. # is this a to a local account?
  456. if domainMatch in recipient:
  457. # get the handle for the local account
  458. nickname = recipient.split(domainMatch)[1]
  459. handle = nickname+'@'+domain
  460. if os.path.isdir(baseDir + '/accounts/' + handle):
  461. # are capabilities granted for this account to the
  462. # sender (actor) of the post?
  463. ocapFilename = \
  464. baseDir + '/accounts/' + handle + \
  465. '/ocap/accept/' + actor.replace('/', '#') + '.json'
  466. if os.path.isfile(ocapFilename):
  467. # read the granted capabilities and obtain the id
  468. ocapJson = loadJson(ocapFilename)
  469. if ocapJson:
  470. if ocapJson.get('id'):
  471. # append with the capabilities id
  472. recipientsDict[handle] = ocapJson['id']
  473. else:
  474. recipientsDict[handle] = None
  475. else:
  476. if debug:
  477. print('DEBUG: ' + ocapFilename + ' not found')
  478. recipientsDict[handle] = None
  479. else:
  480. if debug:
  481. print('DEBUG: ' + baseDir + '/accounts/' +
  482. handle + ' does not exist')
  483. else:
  484. if debug:
  485. print('DEBUG: ' + recipient + ' is not local to ' +
  486. domainMatch)
  487. print(str(toList))
  488. if recipient.endswith('followers'):
  489. if debug:
  490. print('DEBUG: followers detected as post recipients')
  491. followerRecipients = True
  492. return followerRecipients, recipientsDict
  493. def inboxPostRecipients(baseDir: str, postJsonObject: {},
  494. httpPrefix: str, domain: str, port: int,
  495. debug: bool) -> ([], []):
  496. """Returns dictionaries containing the recipients of the given post
  497. The shared dictionary contains followers
  498. """
  499. recipientsDict = {}
  500. recipientsDictFollowers = {}
  501. if not postJsonObject.get('actor'):
  502. if debug:
  503. pprint(postJsonObject)
  504. print('WARNING: inbox post has no actor')
  505. return recipientsDict, recipientsDictFollowers
  506. if ':' in domain:
  507. domain = domain.split(':')[0]
  508. domainBase = domain
  509. if port:
  510. if port != 80 and port != 443:
  511. if ':' not in domain:
  512. domain = domain + ':' + str(port)
  513. domainMatch = '/' + domain + '/users/'
  514. actor = postJsonObject['actor']
  515. # first get any specific people which the post is addressed to
  516. followerRecipients = False
  517. if postJsonObject.get('object'):
  518. if isinstance(postJsonObject['object'], dict):
  519. if postJsonObject['object'].get('to'):
  520. if isinstance(postJsonObject['object']['to'], list):
  521. recipientsList = postJsonObject['object']['to']
  522. else:
  523. recipientsList = [postJsonObject['object']['to']]
  524. if debug:
  525. print('DEBUG: resolving "to"')
  526. includesFollowers, recipientsDict = \
  527. inboxPostRecipientsAdd(baseDir, httpPrefix,
  528. recipientsList,
  529. recipientsDict,
  530. domainMatch, domainBase,
  531. actor, debug)
  532. if includesFollowers:
  533. followerRecipients = True
  534. else:
  535. if debug:
  536. print('DEBUG: inbox post has no "to"')
  537. if postJsonObject['object'].get('cc'):
  538. if isinstance(postJsonObject['object']['cc'], list):
  539. recipientsList = postJsonObject['object']['cc']
  540. else:
  541. recipientsList = [postJsonObject['object']['cc']]
  542. includesFollowers, recipientsDict = \
  543. inboxPostRecipientsAdd(baseDir, httpPrefix,
  544. recipientsList,
  545. recipientsDict,
  546. domainMatch, domainBase,
  547. actor, debug)
  548. if includesFollowers:
  549. followerRecipients = True
  550. else:
  551. if debug:
  552. print('DEBUG: inbox post has no cc')
  553. else:
  554. if debug:
  555. if isinstance(postJsonObject['object'], str):
  556. if '/statuses/' in postJsonObject['object']:
  557. print('DEBUG: inbox item is a link to a post')
  558. else:
  559. if '/users/' in postJsonObject['object']:
  560. print('DEBUG: inbox item is a link to an actor')
  561. if postJsonObject.get('to'):
  562. if isinstance(postJsonObject['to'], list):
  563. recipientsList = postJsonObject['to']
  564. else:
  565. recipientsList = [postJsonObject['to']]
  566. includesFollowers, recipientsDict = \
  567. inboxPostRecipientsAdd(baseDir, httpPrefix,
  568. recipientsList,
  569. recipientsDict,
  570. domainMatch, domainBase,
  571. actor, debug)
  572. if includesFollowers:
  573. followerRecipients = True
  574. if postJsonObject.get('cc'):
  575. if isinstance(postJsonObject['cc'], list):
  576. recipientsList = postJsonObject['cc']
  577. else:
  578. recipientsList = [postJsonObject['cc']]
  579. includesFollowers, recipientsDict = \
  580. inboxPostRecipientsAdd(baseDir, httpPrefix,
  581. recipientsList,
  582. recipientsDict,
  583. domainMatch, domainBase,
  584. actor, debug)
  585. if includesFollowers:
  586. followerRecipients = True
  587. if not followerRecipients:
  588. if debug:
  589. print('DEBUG: no followers were resolved')
  590. return recipientsDict, recipientsDictFollowers
  591. # now resolve the followers
  592. recipientsDictFollowers = \
  593. getFollowersOfActor(baseDir, actor, debug)
  594. return recipientsDict, recipientsDictFollowers
  595. def receiveUndoFollow(session, baseDir: str, httpPrefix: str,
  596. port: int, messageJson: {},
  597. federationList: [],
  598. debug: bool) -> bool:
  599. if not messageJson['object'].get('actor'):
  600. if debug:
  601. print('DEBUG: follow request has no actor within object')
  602. return False
  603. if '/users/' not in messageJson['object']['actor'] and \
  604. '/accounts/' not in messageJson['object']['actor'] and \
  605. '/channel/' not in messageJson['object']['actor'] and \
  606. '/profile/' not in messageJson['object']['actor']:
  607. if debug:
  608. print('DEBUG: "users" or "profile" missing ' +
  609. 'from actor within object')
  610. return False
  611. if messageJson['object']['actor'] != messageJson['actor']:
  612. if debug:
  613. print('DEBUG: actors do not match')
  614. return False
  615. nicknameFollower = \
  616. getNicknameFromActor(messageJson['object']['actor'])
  617. if not nicknameFollower:
  618. print('WARN: unable to find nickname in ' +
  619. messageJson['object']['actor'])
  620. return False
  621. domainFollower, portFollower = \
  622. getDomainFromActor(messageJson['object']['actor'])
  623. domainFollowerFull = domainFollower
  624. if portFollower:
  625. if portFollower != 80 and portFollower != 443:
  626. if ':' not in domainFollower:
  627. domainFollowerFull = domainFollower + ':' + str(portFollower)
  628. nicknameFollowing = \
  629. getNicknameFromActor(messageJson['object']['object'])
  630. if not nicknameFollowing:
  631. print('WARN: unable to find nickname in ' +
  632. messageJson['object']['object'])
  633. return False
  634. domainFollowing, portFollowing = \
  635. getDomainFromActor(messageJson['object']['object'])
  636. domainFollowingFull = domainFollowing
  637. if portFollowing:
  638. if portFollowing != 80 and portFollowing != 443:
  639. if ':' not in domainFollowing:
  640. domainFollowingFull = \
  641. domainFollowing + ':' + str(portFollowing)
  642. if unfollowerOfPerson(baseDir,
  643. nicknameFollowing, domainFollowingFull,
  644. nicknameFollower, domainFollowerFull,
  645. debug):
  646. if debug:
  647. print('DEBUG: Follower ' +
  648. nicknameFollower + '@' + domainFollowerFull +
  649. ' was removed')
  650. return True
  651. if debug:
  652. print('DEBUG: Follower ' +
  653. nicknameFollower + '@' + domainFollowerFull +
  654. ' was not removed')
  655. return False
  656. def receiveUndo(session, baseDir: str, httpPrefix: str,
  657. port: int, sendThreads: [], postLog: [],
  658. cachedWebfingers: {}, personCache: {},
  659. messageJson: {}, federationList: [],
  660. debug: bool,
  661. acceptedCaps=["inbox:write", "objects:read"]) -> bool:
  662. """Receives an undo request within the POST section of HTTPServer
  663. """
  664. if not messageJson['type'].startswith('Undo'):
  665. return False
  666. if debug:
  667. print('DEBUG: Undo activity received')
  668. if not messageJson.get('actor'):
  669. if debug:
  670. print('DEBUG: follow request has no actor')
  671. return False
  672. if '/users/' not in messageJson['actor'] and \
  673. '/accounts/' not in messageJson['actor'] and \
  674. '/channel/' not in messageJson['actor'] and \
  675. '/profile/' not in messageJson['actor']:
  676. if debug:
  677. print('DEBUG: "users" or "profile" missing from actor')
  678. return False
  679. if not messageJson.get('object'):
  680. if debug:
  681. print('DEBUG: ' + messageJson['type'] + ' has no object')
  682. return False
  683. if not isinstance(messageJson['object'], dict):
  684. if debug:
  685. print('DEBUG: ' + messageJson['type'] + ' object is not a dict')
  686. return False
  687. if not messageJson['object'].get('type'):
  688. if debug:
  689. print('DEBUG: ' + messageJson['type'] + ' has no object type')
  690. return False
  691. if not messageJson['object'].get('object'):
  692. if debug:
  693. print('DEBUG: ' + messageJson['type'] +
  694. ' has no object within object')
  695. return False
  696. if not isinstance(messageJson['object']['object'], str):
  697. if debug:
  698. print('DEBUG: ' + messageJson['type'] +
  699. ' object within object is not a string')
  700. return False
  701. if messageJson['object']['type'] == 'Follow':
  702. return receiveUndoFollow(session, baseDir, httpPrefix,
  703. port, messageJson,
  704. federationList, debug)
  705. return False
  706. def personReceiveUpdate(baseDir: str,
  707. domain: str, port: int,
  708. updateNickname: str, updateDomain: str,
  709. updatePort: int,
  710. personJson: {}, personCache: {}, debug: bool) -> bool:
  711. """Changes an actor. eg: avatar or display name change
  712. """
  713. if debug:
  714. print('DEBUG: receiving actor update for '+personJson['url'])
  715. domainFull = domain
  716. if port:
  717. if port != 80 and port != 443:
  718. domainFull = domain + ':' + str(port)
  719. updateDomainFull = updateDomain
  720. if updatePort:
  721. if updatePort != 80 and updatePort != 443:
  722. updateDomainFull = updateDomain + ':' + str(updatePort)
  723. actor = updateDomainFull + '/users/' + updateNickname
  724. if actor not in personJson['id']:
  725. actor = updateDomainFull + '/profile/' + updateNickname
  726. if actor not in personJson['id']:
  727. actor = updateDomainFull + '/channel/' + updateNickname
  728. if actor not in personJson['id']:
  729. actor = updateDomainFull + '/accounts/' + updateNickname
  730. if actor not in personJson['id']:
  731. if debug:
  732. print('actor: ' + actor)
  733. print('id: ' + personJson['id'])
  734. print('DEBUG: Actor does not match id')
  735. return False
  736. if updateDomainFull == domainFull:
  737. if debug:
  738. print('DEBUG: You can only receive actor updates ' +
  739. 'for domains other than your own')
  740. return False
  741. if not personJson.get('publicKey'):
  742. if debug:
  743. print('DEBUG: actor update does not contain a public key')
  744. return False
  745. if not personJson['publicKey'].get('publicKeyPem'):
  746. if debug:
  747. print('DEBUG: actor update does not contain a public key Pem')
  748. return False
  749. actorFilename = baseDir + '/cache/actors/' + \
  750. personJson['id'].replace('/', '#') + '.json'
  751. # check that the public keys match.
  752. # If they don't then this may be a nefarious attempt to hack an account
  753. idx = personJson['id']
  754. if personCache.get(idx):
  755. if personCache[idx]['actor']['publicKey']['publicKeyPem'] != \
  756. personJson['publicKey']['publicKeyPem']:
  757. if debug:
  758. print('WARN: Public key does not match when updating actor')
  759. return False
  760. else:
  761. if os.path.isfile(actorFilename):
  762. existingPersonJson = loadJson(actorFilename)
  763. if existingPersonJson:
  764. if existingPersonJson['publicKey']['publicKeyPem'] != \
  765. personJson['publicKey']['publicKeyPem']:
  766. if debug:
  767. print('WARN: Public key does not match ' +
  768. 'cached actor when updating')
  769. return False
  770. # save to cache in memory
  771. storePersonInCache(baseDir, personJson['id'], personJson, personCache)
  772. # save to cache on file
  773. if saveJson(personJson, actorFilename):
  774. print('actor updated for ' + personJson['id'])
  775. # remove avatar if it exists so that it will be refreshed later
  776. # when a timeline is constructed
  777. actorStr = personJson['id'].replace('/', '-')
  778. removeAvatarFromCache(baseDir, actorStr)
  779. return True
  780. def receiveUpdateToQuestion(recentPostsCache: {}, messageJson: {},
  781. baseDir: str, nickname: str, domain: str) -> None:
  782. """Updating a question as new votes arrive
  783. """
  784. # message url of the question
  785. if not messageJson.get('id'):
  786. return
  787. if not messageJson.get('actor'):
  788. return
  789. messageId = messageJson['id'].replace('/activity', '')
  790. if '#' in messageId:
  791. messageId = messageId.split('#', 1)[0]
  792. # find the question post
  793. postFilename = locatePost(baseDir, nickname, domain, messageId)
  794. if not postFilename:
  795. return
  796. # load the json for the question
  797. postJsonObject = loadJson(postFilename, 1)
  798. if not postJsonObject:
  799. return
  800. if not postJsonObject.get('actor'):
  801. return
  802. # does the actor match?
  803. if postJsonObject['actor'] != messageJson['actor']:
  804. return
  805. saveJson(messageJson, postFilename)
  806. # ensure that the cached post is removed if it exists, so
  807. # that it then will be recreated
  808. cachedPostFilename = \
  809. getCachedPostFilename(baseDir, nickname, domain, messageJson)
  810. if cachedPostFilename:
  811. if os.path.isfile(cachedPostFilename):
  812. os.remove(cachedPostFilename)
  813. # remove from memory cache
  814. removePostFromCache(messageJson, recentPostsCache)
  815. def receiveUpdate(recentPostsCache: {}, session, baseDir: str,
  816. httpPrefix: str, domain: str, port: int,
  817. sendThreads: [], postLog: [], cachedWebfingers: {},
  818. personCache: {}, messageJson: {}, federationList: [],
  819. nickname: str, debug: bool) -> bool:
  820. """Receives an Update activity within the POST section of HTTPServer
  821. """
  822. if messageJson['type'] != 'Update':
  823. return False
  824. if not messageJson.get('actor'):
  825. if debug:
  826. print('DEBUG: ' + messageJson['type'] + ' has no actor')
  827. return False
  828. if not messageJson.get('object'):
  829. if debug:
  830. print('DEBUG: ' + messageJson['type'] + ' has no object')
  831. return False
  832. if not isinstance(messageJson['object'], dict):
  833. if debug:
  834. print('DEBUG: ' + messageJson['type'] + ' object is not a dict')
  835. return False
  836. if not messageJson['object'].get('type'):
  837. if debug:
  838. print('DEBUG: ' + messageJson['type'] + ' object has no type')
  839. return False
  840. if '/users/' not in messageJson['actor'] and \
  841. '/accounts/' not in messageJson['actor'] and \
  842. '/channel/' not in messageJson['actor'] and \
  843. '/profile/' not in messageJson['actor']:
  844. if debug:
  845. print('DEBUG: "users" or "profile" missing from actor in ' +
  846. messageJson['type'])
  847. return False
  848. if messageJson['object']['type'] == 'Question':
  849. receiveUpdateToQuestion(recentPostsCache, messageJson,
  850. baseDir, nickname, domain)
  851. if debug:
  852. print('DEBUG: Question update was received')
  853. return True
  854. if messageJson['type'] == 'Person':
  855. if messageJson.get('url') and messageJson.get('id'):
  856. print('Request to update unwrapped actor: ' + messageJson['id'])
  857. updateNickname = getNicknameFromActor(messageJson['id'])
  858. if updateNickname:
  859. updateDomain, updatePort = \
  860. getDomainFromActor(messageJson['id'])
  861. if personReceiveUpdate(baseDir, domain, port,
  862. updateNickname, updateDomain,
  863. updatePort, messageJson,
  864. personCache, debug):
  865. if debug:
  866. print('DEBUG: ' +
  867. 'Unwrapped profile update was received for ' +
  868. messageJson['url'])
  869. return True
  870. if messageJson['object']['type'] == 'Person' or \
  871. messageJson['object']['type'] == 'Application' or \
  872. messageJson['object']['type'] == 'Group' or \
  873. messageJson['object']['type'] == 'Service':
  874. if messageJson['object'].get('url') and \
  875. messageJson['object'].get('id'):
  876. print('Request to update actor: ' + messageJson['actor'])
  877. updateNickname = getNicknameFromActor(messageJson['actor'])
  878. if updateNickname:
  879. updateDomain, updatePort = \
  880. getDomainFromActor(messageJson['actor'])
  881. if personReceiveUpdate(baseDir,
  882. domain, port,
  883. updateNickname, updateDomain,
  884. updatePort,
  885. messageJson['object'],
  886. personCache, debug):
  887. if debug:
  888. print('DEBUG: Profile update was received for ' +
  889. messageJson['object']['url'])
  890. return True
  891. if messageJson['object'].get('capability') and \
  892. messageJson['object'].get('scope'):
  893. nickname = getNicknameFromActor(messageJson['object']['scope'])
  894. if nickname:
  895. domain, tempPort = \
  896. getDomainFromActor(messageJson['object']['scope'])
  897. if messageJson['object']['type'] == 'Capability':
  898. capability = messageJson['object']['capability']
  899. if capabilitiesReceiveUpdate(baseDir, nickname, domain, port,
  900. messageJson['actor'],
  901. messageJson['object']['id'],
  902. capability,
  903. debug):
  904. if debug:
  905. print('DEBUG: An update was received')
  906. return True
  907. return False
  908. def receiveLike(recentPostsCache: {},
  909. session, handle: str, isGroup: bool, baseDir: str,
  910. httpPrefix: str, domain: str, port: int,
  911. onionDomain: str,
  912. sendThreads: [], postLog: [], cachedWebfingers: {},
  913. personCache: {}, messageJson: {}, federationList: [],
  914. debug: bool) -> bool:
  915. """Receives a Like activity within the POST section of HTTPServer
  916. """
  917. if messageJson['type'] != 'Like':
  918. return False
  919. if not messageJson.get('actor'):
  920. if debug:
  921. print('DEBUG: ' + messageJson['type'] + ' has no actor')
  922. return False
  923. if not messageJson.get('object'):
  924. if debug:
  925. print('DEBUG: ' + messageJson['type'] + ' has no object')
  926. return False
  927. if not isinstance(messageJson['object'], str):
  928. if debug:
  929. print('DEBUG: ' + messageJson['type'] + ' object is not a string')
  930. return False
  931. if not messageJson.get('to'):
  932. if debug:
  933. print('DEBUG: ' + messageJson['type'] + ' has no "to" list')
  934. return False
  935. if '/users/' not in messageJson['actor'] and \
  936. '/accounts/' not in messageJson['actor'] and \
  937. '/channel/' not in messageJson['actor'] and \
  938. '/profile/' not in messageJson['actor']:
  939. if debug:
  940. print('DEBUG: "users" or "profile" missing from actor in ' +
  941. messageJson['type'])
  942. return False
  943. if '/statuses/' not in messageJson['object']:
  944. if debug:
  945. print('DEBUG: "statuses" missing from object in ' +
  946. messageJson['type'])
  947. return False
  948. if not os.path.isdir(baseDir + '/accounts/' + handle):
  949. print('DEBUG: unknown recipient of like - ' + handle)
  950. # if this post in the outbox of the person?
  951. postFilename = locatePost(baseDir, handle.split('@')[0],
  952. handle.split('@')[1],
  953. messageJson['object'])
  954. if not postFilename:
  955. if debug:
  956. print('DEBUG: post not found in inbox or outbox')
  957. print(messageJson['object'])
  958. return True
  959. if debug:
  960. print('DEBUG: liked post found in inbox')
  961. if not alreadyLiked(baseDir,
  962. handle.split('@')[0],
  963. handle.split('@')[1],
  964. messageJson['object'],
  965. messageJson['actor']):
  966. updateLikesCollection(recentPostsCache, baseDir, postFilename,
  967. messageJson['object'],
  968. messageJson['actor'], domain, debug)
  969. likeNotify(baseDir, domain, onionDomain, handle,
  970. messageJson['actor'], messageJson['object'])
  971. return True
  972. def receiveUndoLike(recentPostsCache: {},
  973. session, handle: str, isGroup: bool, baseDir: str,
  974. httpPrefix: str, domain: str, port: int,
  975. sendThreads: [], postLog: [], cachedWebfingers: {},
  976. personCache: {}, messageJson: {}, federationList: [],
  977. debug: bool) -> bool:
  978. """Receives an undo like activity within the POST section of HTTPServer
  979. """
  980. if messageJson['type'] != 'Undo':
  981. return False
  982. if not messageJson.get('actor'):
  983. return False
  984. if not messageJson.get('object'):
  985. return False
  986. if not isinstance(messageJson['object'], dict):
  987. return False
  988. if not messageJson['object'].get('type'):
  989. return False
  990. if messageJson['object']['type'] != 'Like':
  991. return False
  992. if not messageJson['object'].get('object'):
  993. if debug:
  994. print('DEBUG: ' + messageJson['type'] + ' like has no object')
  995. return False
  996. if not isinstance(messageJson['object']['object'], str):
  997. if debug:
  998. print('DEBUG: ' + messageJson['type'] +
  999. ' like object is not a string')
  1000. return False
  1001. if '/users/' not in messageJson['actor'] and \
  1002. '/accounts/' not in messageJson['actor'] and \
  1003. '/channel/' not in messageJson['actor'] and \
  1004. '/profile/' not in messageJson['actor']:
  1005. if debug:
  1006. print('DEBUG: "users" or "profile" missing from actor in ' +
  1007. messageJson['type'] + ' like')
  1008. return False
  1009. if '/statuses/' not in messageJson['object']['object']:
  1010. if debug:
  1011. print('DEBUG: "statuses" missing from like object in ' +
  1012. messageJson['type'])
  1013. return False
  1014. if not os.path.isdir(baseDir + '/accounts/' + handle):
  1015. print('DEBUG: unknown recipient of undo like - ' + handle)
  1016. # if this post in the outbox of the person?
  1017. postFilename = \
  1018. locatePost(baseDir, handle.split('@')[0], handle.split('@')[1],
  1019. messageJson['object']['object'])
  1020. if not postFilename:
  1021. if debug:
  1022. print('DEBUG: unliked post not found in inbox or outbox')
  1023. print(messageJson['object']['object'])
  1024. return True
  1025. if debug:
  1026. print('DEBUG: liked post found in inbox. Now undoing.')
  1027. undoLikesCollectionEntry(recentPostsCache, baseDir, postFilename,
  1028. messageJson['object'],
  1029. messageJson['actor'], domain, debug)
  1030. return True
  1031. def receiveBookmark(recentPostsCache: {},
  1032. session, handle: str, isGroup: bool, baseDir: str,
  1033. httpPrefix: str, domain: str, port: int,
  1034. sendThreads: [], postLog: [], cachedWebfingers: {},
  1035. personCache: {}, messageJson: {}, federationList: [],
  1036. debug: bool) -> bool:
  1037. """Receives a bookmark activity within the POST section of HTTPServer
  1038. """
  1039. if messageJson['type'] != 'Bookmark':
  1040. return False
  1041. if not messageJson.get('actor'):
  1042. if debug:
  1043. print('DEBUG: ' + messageJson['type'] + ' has no actor')
  1044. return False
  1045. if not messageJson.get('object'):
  1046. if debug:
  1047. print('DEBUG: ' + messageJson['type'] + ' has no object')
  1048. return False
  1049. if not isinstance(messageJson['object'], str):
  1050. if debug:
  1051. print('DEBUG: ' + messageJson['type'] + ' object is not a string')
  1052. return False
  1053. if not messageJson.get('to'):
  1054. if debug:
  1055. print('DEBUG: ' + messageJson['type'] + ' has no "to" list')
  1056. return False
  1057. if '/users/' not in messageJson['actor']:
  1058. if debug:
  1059. print('DEBUG: "users" missing from actor in ' +
  1060. messageJson['type'])
  1061. return False
  1062. if '/statuses/' not in messageJson['object']:
  1063. if debug:
  1064. print('DEBUG: "statuses" missing from object in ' +
  1065. messageJson['type'])
  1066. return False
  1067. if domain not in handle.split('@')[1]:
  1068. if debug:
  1069. print('DEBUG: unrecognized domain ' + handle)
  1070. return False
  1071. domainFull = domain
  1072. if port:
  1073. if port != 80 and port != 443:
  1074. domainFull = domain + ':' + str(port)
  1075. nickname = handle.split('@')[0]
  1076. if not messageJson['actor'].endswith(domainFull + '/users/' + nickname):
  1077. if debug:
  1078. print('DEBUG: ' +
  1079. 'bookmark actor should be the same as the handle sent to ' +
  1080. handle + ' != ' + messageJson['actor'])
  1081. return False
  1082. if not os.path.isdir(baseDir + '/accounts/' + handle):
  1083. print('DEBUG: unknown recipient of bookmark - ' + handle)
  1084. # if this post in the outbox of the person?
  1085. postFilename = locatePost(baseDir, nickname, domain, messageJson['object'])
  1086. if not postFilename:
  1087. if debug:
  1088. print('DEBUG: post not found in inbox or outbox')
  1089. print(messageJson['object'])
  1090. return True
  1091. if debug:
  1092. print('DEBUG: bookmarked post was found')
  1093. updateBookmarksCollection(recentPostsCache, baseDir, postFilename,
  1094. messageJson['object'],
  1095. messageJson['actor'], domain, debug)
  1096. return True
  1097. def receiveUndoBookmark(recentPostsCache: {},
  1098. session, handle: str, isGroup: bool, baseDir: str,
  1099. httpPrefix: str, domain: str, port: int,
  1100. sendThreads: [], postLog: [], cachedWebfingers: {},
  1101. personCache: {}, messageJson: {}, federationList: [],
  1102. debug: bool) -> bool:
  1103. """Receives an undo bookmark activity within the POST section of HTTPServer
  1104. """
  1105. if messageJson['type'] != 'Undo':
  1106. return False
  1107. if not messageJson.get('actor'):
  1108. return False
  1109. if not messageJson.get('object'):
  1110. return False
  1111. if not isinstance(messageJson['object'], dict):
  1112. return False
  1113. if not messageJson['object'].get('type'):
  1114. return False
  1115. if messageJson['object']['type'] != 'Bookmark':
  1116. return False
  1117. if not messageJson['object'].get('object'):
  1118. if debug:
  1119. print('DEBUG: ' + messageJson['type'] + ' like has no object')
  1120. return False
  1121. if not isinstance(messageJson['object']['object'], str):
  1122. if debug:
  1123. print('DEBUG: ' + messageJson['type'] +
  1124. ' like object is not a string')
  1125. return False
  1126. if '/users/' not in messageJson['actor']:
  1127. if debug:
  1128. print('DEBUG: "users" missing from actor in ' +
  1129. messageJson['type'] + ' like')
  1130. return False
  1131. if '/statuses/' not in messageJson['object']['object']:
  1132. if debug:
  1133. print('DEBUG: "statuses" missing from like object in ' +
  1134. messageJson['type'])
  1135. return False
  1136. domainFull = domain
  1137. if port:
  1138. if port != 80 and port != 443:
  1139. domainFull = domain + ':' + str(port)
  1140. nickname = handle.split('@')[0]
  1141. if domain not in handle.split('@')[1]:
  1142. if debug:
  1143. print('DEBUG: unrecognized bookmark domain ' + handle)
  1144. return False
  1145. if not messageJson['actor'].endswith(domainFull + '/users/' + nickname):
  1146. if debug:
  1147. print('DEBUG: ' +
  1148. 'bookmark actor should be the same as the handle sent to ' +
  1149. handle + ' != ' + messageJson['actor'])
  1150. return False
  1151. if not os.path.isdir(baseDir + '/accounts/' + handle):
  1152. print('DEBUG: unknown recipient of bookmark undo - ' + handle)
  1153. # if this post in the outbox of the person?
  1154. postFilename = locatePost(baseDir, nickname, domain,
  1155. messageJson['object']['object'])
  1156. if not postFilename:
  1157. if debug:
  1158. print('DEBUG: unbookmarked post not found in inbox or outbox')
  1159. print(messageJson['object']['object'])
  1160. return True
  1161. if debug:
  1162. print('DEBUG: bookmarked post found. Now undoing.')
  1163. undoBookmarksCollectionEntry(recentPostsCache, baseDir, postFilename,
  1164. messageJson['object'],
  1165. messageJson['actor'], domain, debug)
  1166. return True
  1167. def receiveDelete(session, handle: str, isGroup: bool, baseDir: str,
  1168. httpPrefix: str, domain: str, port: int,
  1169. sendThreads: [], postLog: [], cachedWebfingers: {},
  1170. personCache: {}, messageJson: {}, federationList: [],
  1171. debug: bool, allowDeletion: bool,
  1172. recentPostsCache: {}) -> bool:
  1173. """Receives a Delete activity within the POST section of HTTPServer
  1174. """
  1175. if messageJson['type'] != 'Delete':
  1176. return False
  1177. if not messageJson.get('actor'):
  1178. if debug:
  1179. print('DEBUG: ' + messageJson['type'] + ' has no actor')
  1180. return False
  1181. if debug:
  1182. print('DEBUG: Delete activity arrived')
  1183. if not messageJson.get('object'):
  1184. if debug:
  1185. print('DEBUG: ' + messageJson['type'] + ' has no object')
  1186. return False
  1187. if not isinstance(messageJson['object'], str):
  1188. if debug:
  1189. print('DEBUG: ' + messageJson['type'] + ' object is not a string')
  1190. return False
  1191. domainFull = domain
  1192. if port:
  1193. if port != 80 and port != 443:
  1194. if ':' not in domain:
  1195. domainFull = domain + ':' + str(port)
  1196. deletePrefix = httpPrefix + '://' + domainFull + '/'
  1197. if (not allowDeletion and
  1198. (not messageJson['object'].startswith(deletePrefix) or
  1199. not messageJson['actor'].startswith(deletePrefix))):
  1200. if debug:
  1201. print('DEBUG: delete not permitted from other instances')
  1202. return False
  1203. if not messageJson.get('to'):
  1204. if debug:
  1205. print('DEBUG: ' + messageJson['type'] + ' has no "to" list')
  1206. return False
  1207. if '/users/' not in messageJson['actor'] and \
  1208. '/accounts/' not in messageJson['actor'] and \
  1209. '/channel/' not in messageJson['actor'] and \
  1210. '/profile/' not in messageJson['actor']:
  1211. if debug:
  1212. print('DEBUG: ' +
  1213. '"users" or "profile" missing from actor in ' +
  1214. messageJson['type'])
  1215. return False
  1216. if '/statuses/' not in messageJson['object']:
  1217. if debug:
  1218. print('DEBUG: "statuses" missing from object in ' +
  1219. messageJson['type'])
  1220. return False
  1221. if messageJson['actor'] not in messageJson['object']:
  1222. if debug:
  1223. print('DEBUG: actor is not the owner of the post to be deleted')
  1224. if not os.path.isdir(baseDir + '/accounts/' + handle):
  1225. print('DEBUG: unknown recipient of like - ' + handle)
  1226. # if this post in the outbox of the person?
  1227. messageId = messageJson['object'].replace('/activity', '')
  1228. messageId = messageId.replace('/undo', '')
  1229. removeModerationPostFromIndex(baseDir, messageId, debug)
  1230. postFilename = locatePost(baseDir, handle.split('@')[0],
  1231. handle.split('@')[1], messageId)
  1232. if not postFilename:
  1233. if debug:
  1234. print('DEBUG: delete post not found in inbox or outbox')
  1235. print(messageId)
  1236. return True
  1237. deletePost(baseDir, httpPrefix, handle.split('@')[0],
  1238. handle.split('@')[1], postFilename, debug,
  1239. recentPostsCache)
  1240. if debug:
  1241. print('DEBUG: post deleted - ' + postFilename)
  1242. return True
  1243. def receiveAnnounce(recentPostsCache: {},
  1244. session, handle: str, isGroup: bool, baseDir: str,
  1245. httpPrefix: str, domain: str, onionDomain: str, port: int,
  1246. sendThreads: [], postLog: [], cachedWebfingers: {},
  1247. personCache: {}, messageJson: {}, federationList: [],
  1248. debug: bool, translate: {},
  1249. YTReplacementDomain: str) -> bool:
  1250. """Receives an announce activity within the POST section of HTTPServer
  1251. """
  1252. if messageJson['type'] != 'Announce':
  1253. return False
  1254. if '@' not in handle:
  1255. if debug:
  1256. print('DEBUG: bad handle ' + handle)
  1257. return False
  1258. if not messageJson.get('actor'):
  1259. if debug:
  1260. print('DEBUG: ' + messageJson['type'] + ' has no actor')
  1261. return False
  1262. if debug:
  1263. print('DEBUG: receiving announce on ' + handle)
  1264. if not messageJson.get('object'):
  1265. if debug:
  1266. print('DEBUG: ' + messageJson['type'] + ' has no object')
  1267. return False
  1268. if not isinstance(messageJson['object'], str):
  1269. if debug:
  1270. print('DEBUG: ' + messageJson['type'] + ' object is not a string')
  1271. return False
  1272. if not messageJson.get('to'):
  1273. if debug:
  1274. print('DEBUG: ' + messageJson['type'] + ' has no "to" list')
  1275. return False
  1276. if '/users/' not in messageJson['actor'] and \
  1277. '/accounts/' not in messageJson['actor'] and \
  1278. '/channel/' not in messageJson['actor'] and \
  1279. '/profile/' not in messageJson['actor']:
  1280. if debug:
  1281. print('DEBUG: ' +
  1282. '"users" or "profile" missing from actor in ' +
  1283. messageJson['type'])
  1284. return False
  1285. if '/users/' not in messageJson['object'] and \
  1286. '/accounts/' not in messageJson['object'] and \
  1287. '/channel/' not in messageJson['object'] and \
  1288. '/profile/' not in messageJson['object']:
  1289. if debug:
  1290. print('DEBUG: ' +
  1291. '"users", "channel" or "profile" missing in ' +
  1292. messageJson['type'])
  1293. return False
  1294. prefixes = getProtocolPrefixes()
  1295. # is the domain of the announce actor blocked?
  1296. objectDomain = messageJson['object']
  1297. for prefix in prefixes:
  1298. objectDomain = objectDomain.replace(prefix, '')
  1299. if '/' in objectDomain:
  1300. objectDomain = objectDomain.split('/')[0]
  1301. if isBlockedDomain(baseDir, objectDomain):
  1302. if debug:
  1303. print('DEBUG: announced domain is blocked')
  1304. return False
  1305. if not os.path.isdir(baseDir + '/accounts/' + handle):
  1306. print('DEBUG: unknown recipient of announce - ' + handle)
  1307. # is the announce actor blocked?
  1308. nickname = handle.split('@')[0]
  1309. actorNickname = getNicknameFromActor(messageJson['actor'])
  1310. actorDomain, actorPort = getDomainFromActor(messageJson['actor'])
  1311. if isBlocked(baseDir, nickname, domain, actorNickname, actorDomain):
  1312. print('Receive announce blocked for actor: ' +
  1313. actorNickname + '@' + actorDomain)
  1314. return False
  1315. # is this post in the outbox of the person?
  1316. postFilename = locatePost(baseDir, nickname, domain,
  1317. messageJson['object'])
  1318. if not postFilename:
  1319. if debug:
  1320. print('DEBUG: announce post not found in inbox or outbox')
  1321. print(messageJson['object'])
  1322. return True
  1323. updateAnnounceCollection(recentPostsCache, baseDir, postFilename,
  1324. messageJson['actor'], domain, debug)
  1325. if debug:
  1326. print('DEBUG: Downloading announce post ' + messageJson['actor'] +
  1327. ' -> ' + messageJson['object'])
  1328. postJsonObject = downloadAnnounce(session, baseDir, httpPrefix,
  1329. nickname, domain, messageJson,
  1330. __version__, translate,
  1331. YTReplacementDomain)
  1332. if postJsonObject:
  1333. if debug:
  1334. print('DEBUG: Announce post downloaded for ' +
  1335. messageJson['actor'] + ' -> ' + messageJson['object'])
  1336. storeHashTags(baseDir, nickname, postJsonObject)
  1337. # Try to obtain the actor for this person
  1338. # so that their avatar can be shown
  1339. lookupActor = None
  1340. if postJsonObject.get('attributedTo'):
  1341. if isinstance(postJsonObject['attributedTo'], str):
  1342. lookupActor = postJsonObject['attributedTo']
  1343. else:
  1344. if postJsonObject.get('object'):
  1345. if isinstance(postJsonObject['object'], dict):
  1346. if postJsonObject['object'].get('attributedTo'):
  1347. attrib = postJsonObject['object']['attributedTo']
  1348. if isinstance(attrib, str):
  1349. lookupActor = attrib
  1350. if lookupActor:
  1351. if '/users/' in lookupActor or \
  1352. '/accounts/' in lookupActor or \
  1353. '/channel/' in lookupActor or \
  1354. '/profile/' in lookupActor:
  1355. if '/statuses/' in lookupActor:
  1356. lookupActor = lookupActor.split('/statuses/')[0]
  1357. if debug:
  1358. print('DEBUG: Obtaining actor for announce post ' +
  1359. lookupActor)
  1360. for tries in range(6):
  1361. pubKey = \
  1362. getPersonPubKey(baseDir, session, lookupActor,
  1363. personCache, debug,
  1364. __version__, httpPrefix,
  1365. domain, onionDomain)
  1366. if pubKey:
  1367. print('DEBUG: public key obtained for announce: ' +
  1368. lookupActor)
  1369. break
  1370. if debug:
  1371. print('DEBUG: Retry ' + str(tries + 1) +
  1372. ' obtaining actor for ' + lookupActor)
  1373. time.sleep(5)
  1374. if debug:
  1375. print('DEBUG: announced/repeated post arrived in inbox')
  1376. return True
  1377. def receiveUndoAnnounce(recentPostsCache: {},
  1378. session, handle: str, isGroup: bool, baseDir: str,
  1379. httpPrefix: str, domain: str, port: int,
  1380. sendThreads: [], postLog: [], cachedWebfingers: {},
  1381. personCache: {}, messageJson: {}, federationList: [],
  1382. debug: bool) -> bool:
  1383. """Receives an undo announce activity within the POST section of HTTPServer
  1384. """
  1385. if messageJson['type'] != 'Undo':
  1386. return False
  1387. if not messageJson.get('actor'):
  1388. return False
  1389. if not messageJson.get('object'):
  1390. return False
  1391. if not isinstance(messageJson['object'], dict):
  1392. return False
  1393. if not messageJson['object'].get('object'):
  1394. return False
  1395. if not isinstance(messageJson['object']['object'], str):
  1396. return False
  1397. if messageJson['object']['type'] != 'Announce':
  1398. return False
  1399. if '/users/' not in messageJson['actor'] and \
  1400. '/accounts/' not in messageJson['actor'] and \
  1401. '/channel/' not in messageJson['actor'] and \
  1402. '/profile/' not in messageJson['actor']:
  1403. if debug:
  1404. print('DEBUG: "users" or "profile" missing from actor in ' +
  1405. messageJson['type'] + ' announce')
  1406. return False
  1407. if not os.path.isdir(baseDir + '/accounts/' + handle):
  1408. print('DEBUG: unknown recipient of undo announce - ' + handle)
  1409. # if this post in the outbox of the person?
  1410. postFilename = locatePost(baseDir, handle.split('@')[0],
  1411. handle.split('@')[1],
  1412. messageJson['object']['object'])
  1413. if not postFilename:
  1414. if debug:
  1415. print('DEBUG: undo announce post not found in inbox or outbox')
  1416. print(messageJson['object']['object'])
  1417. return True
  1418. if debug:
  1419. print('DEBUG: announced/repeated post to be undone found in inbox')
  1420. postJsonObject = loadJson(postFilename)
  1421. if postJsonObject:
  1422. if not postJsonObject.get('type'):
  1423. if postJsonObject['type'] != 'Announce':
  1424. if debug:
  1425. print("DEBUG: Attempt to undo something " +
  1426. "which isn't an announcement")
  1427. return False
  1428. undoAnnounceCollectionEntry(recentPostsCache, baseDir, postFilename,
  1429. messageJson['actor'], domain, debug)
  1430. if os.path.isfile(postFilename):
  1431. os.remove(postFilename)
  1432. return True
  1433. def populateReplies(baseDir: str, httpPrefix: str, domain: str,
  1434. messageJson: {}, maxReplies: int, debug: bool) -> bool:
  1435. """Updates the list of replies for a post on this domain if
  1436. a reply to it arrives
  1437. """
  1438. if not messageJson.get('id'):
  1439. return False
  1440. if not messageJson.get('object'):
  1441. return False
  1442. if not isinstance(messageJson['object'], dict):
  1443. return False
  1444. if not messageJson['object'].get('inReplyTo'):
  1445. return False
  1446. if not messageJson['object'].get('to'):
  1447. return False
  1448. replyTo = messageJson['object']['inReplyTo']
  1449. if debug:
  1450. print('DEBUG: post contains a reply')
  1451. # is this a reply to a post on this domain?
  1452. if not replyTo.startswith(httpPrefix + '://' + domain + '/'):
  1453. if debug:
  1454. print('DEBUG: post is a reply to another not on this domain')
  1455. print(replyTo)
  1456. print('Expected: ' + httpPrefix + '://' + domain + '/')
  1457. return False
  1458. replyToNickname = getNicknameFromActor(replyTo)
  1459. if not replyToNickname:
  1460. print('DEBUG: no nickname found for ' + replyTo)
  1461. return False
  1462. replyToDomain, replyToPort = getDomainFromActor(replyTo)
  1463. if not replyToDomain:
  1464. if debug:
  1465. print('DEBUG: no domain found for ' + replyTo)
  1466. return False
  1467. postFilename = locatePost(baseDir, replyToNickname,
  1468. replyToDomain, replyTo)
  1469. if not postFilename:
  1470. if debug:
  1471. print('DEBUG: post may have expired - ' + replyTo)
  1472. return False
  1473. # populate a text file containing the ids of replies
  1474. postRepliesFilename = postFilename.replace('.json', '.replies')
  1475. messageId = messageJson['id'].replace('/activity', '')
  1476. messageId = messageId.replace('/undo', '')
  1477. if os.path.isfile(postRepliesFilename):
  1478. numLines = sum(1 for line in open(postRepliesFilename))
  1479. if numLines > maxReplies:
  1480. return False
  1481. if messageId not in open(postRepliesFilename).read():
  1482. repliesFile = open(postRepliesFilename, "a")
  1483. repliesFile.write(messageId + '\n')
  1484. repliesFile.close()
  1485. else:
  1486. repliesFile = open(postRepliesFilename, "w")
  1487. repliesFile.write(messageId + '\n')
  1488. repliesFile.close()
  1489. return True
  1490. def estimateNumberOfMentions(content: str) -> int:
  1491. """Returns a rough estimate of the number of mentions
  1492. """
  1493. return int(content.count('@') / 2)
  1494. def estimateNumberOfEmoji(content: str) -> int:
  1495. """Returns a rough estimate of the number of emoji
  1496. """
  1497. return int(content.count(':') / 2)
  1498. def validPostContent(baseDir: str, nickname: str, domain: str,
  1499. messageJson: {}, maxMentions: int, maxEmoji: int) -> bool:
  1500. """Is the content of a received post valid?
  1501. Check for bad html
  1502. Check for hellthreads
  1503. Check number of tags is reasonable
  1504. """
  1505. if not messageJson.get('object'):
  1506. return True
  1507. if not isinstance(messageJson['object'], dict):
  1508. return True
  1509. if not messageJson['object'].get('content'):
  1510. return True
  1511. if not messageJson['object'].get('published'):
  1512. return False
  1513. if 'T' not in messageJson['object']['published']:
  1514. return False
  1515. if 'Z' not in messageJson['object']['published']:
  1516. return False
  1517. if isGitPatch(baseDir, nickname, domain,
  1518. messageJson['object']['type'],
  1519. messageJson['object']['summary'],
  1520. messageJson['object']['content']):
  1521. return True
  1522. if dangerousMarkup(messageJson['object']['content']):
  1523. if messageJson['object'].get('id'):
  1524. print('REJECT ARBITRARY HTML: ' + messageJson['object']['id'])
  1525. print('REJECT ARBITRARY HTML: bad string in post - ' +
  1526. messageJson['object']['content'])
  1527. return False
  1528. # check (rough) number of mentions
  1529. mentionsEst = estimateNumberOfMentions(messageJson['object']['content'])
  1530. if mentionsEst > maxMentions:
  1531. if messageJson['object'].get('id'):
  1532. print('REJECT HELLTHREAD: ' + messageJson['object']['id'])
  1533. print('REJECT HELLTHREAD: Too many mentions in post - ' +
  1534. messageJson['object']['content'])
  1535. return False
  1536. if estimateNumberOfEmoji(messageJson['object']['content']) > maxEmoji:
  1537. if messageJson['object'].get('id'):
  1538. print('REJECT EMOJI OVERLOAD: ' + messageJson['object']['id'])
  1539. print('REJECT EMOJI OVERLOAD: Too many emoji in post - ' +
  1540. messageJson['object']['content'])
  1541. return False
  1542. # check number of tags
  1543. if messageJson['object'].get('tag'):
  1544. if not isinstance(messageJson['object']['tag'], list):
  1545. messageJson['object']['tag'] = []
  1546. else:
  1547. if len(messageJson['object']['tag']) > int(maxMentions * 2):
  1548. if messageJson['object'].get('id'):
  1549. print('REJECT: ' + messageJson['object']['id'])
  1550. print('REJECT: Too many tags in post - ' +
  1551. messageJson['object']['tag'])
  1552. return False
  1553. # check for filtered content
  1554. if isFiltered(baseDir, nickname, domain,
  1555. messageJson['object']['content']):
  1556. print('REJECT: content filtered')
  1557. return False
  1558. print('ACCEPT: post content is valid')
  1559. return True
  1560. def obtainAvatarForReplyPost(session, baseDir: str, httpPrefix: str,
  1561. domain: str, onionDomain: str, personCache: {},
  1562. postJsonObject: {}, debug: bool) -> None:
  1563. """Tries to obtain the actor for the person being replied to
  1564. so that their avatar can later be shown
  1565. """
  1566. if not postJsonObject.get('object'):
  1567. return
  1568. if not isinstance(postJsonObject['object'], dict):
  1569. return
  1570. if not postJsonObject['object'].get('inReplyTo'):
  1571. return
  1572. lookupActor = postJsonObject['object']['inReplyTo']
  1573. if not lookupActor:
  1574. return
  1575. if not ('/users/' in lookupActor or
  1576. '/accounts/' in lookupActor or
  1577. '/channel/' in lookupActor or
  1578. '/profile/' in lookupActor):
  1579. return
  1580. if '/statuses/' in lookupActor:
  1581. lookupActor = lookupActor.split('/statuses/')[0]
  1582. if debug:
  1583. print('DEBUG: Obtaining actor for reply post ' + lookupActor)
  1584. for tries in range(6):
  1585. pubKey = \
  1586. getPersonPubKey(baseDir, session, lookupActor,
  1587. personCache, debug,
  1588. __version__, httpPrefix,
  1589. domain, onionDomain)
  1590. if pubKey:
  1591. print('DEBUG: public key obtained for reply: ' + lookupActor)
  1592. break
  1593. if debug:
  1594. print('DEBUG: Retry ' + str(tries + 1) +
  1595. ' obtaining actor for ' + lookupActor)
  1596. time.sleep(5)
  1597. def dmNotify(baseDir: str, handle: str, url: str) -> None:
  1598. """Creates a notification that a new DM has arrived
  1599. """
  1600. accountDir = baseDir + '/accounts/' + handle
  1601. if not os.path.isdir(accountDir):
  1602. return
  1603. dmFile = accountDir + '/.newDM'
  1604. if not os.path.isfile(dmFile):
  1605. with open(dmFile, 'w+') as fp:
  1606. fp.write(url)
  1607. def alreadyLiked(baseDir: str, nickname: str, domain: str,
  1608. postUrl: str, likerActor: str) -> bool:
  1609. """Is the given post already liked by the given handle?
  1610. """
  1611. postFilename = \
  1612. locatePost(baseDir, nickname, domain, postUrl)
  1613. if not postFilename:
  1614. return False
  1615. postJsonObject = loadJson(postFilename, 1)
  1616. if not postJsonObject:
  1617. return False
  1618. if not postJsonObject.get('object'):
  1619. return False
  1620. if not isinstance(postJsonObject['object'], dict):
  1621. return False
  1622. if not postJsonObject['object'].get('likes'):
  1623. return False
  1624. if not postJsonObject['object']['likes'].get('items'):
  1625. return False
  1626. for like in postJsonObject['object']['likes']['items']:
  1627. if not like.get('type'):
  1628. continue
  1629. if not like.get('actor'):
  1630. continue
  1631. if like['type'] != 'Like':
  1632. continue
  1633. if like['actor'] == likerActor:
  1634. return True
  1635. return False
  1636. def likeNotify(baseDir: str, domain: str, onionDomain: str,
  1637. handle: str, actor: str, url: str) -> None:
  1638. """Creates a notification that a like has arrived
  1639. """
  1640. # This is not you liking your own post
  1641. if actor in url:
  1642. return
  1643. # check that the liked post was by this handle
  1644. nickname = handle.split('@')[0]
  1645. if '/' + domain + '/users/' + nickname not in url:
  1646. if not onionDomain:
  1647. return
  1648. if '/' + onionDomain + '/users/' + nickname not in url:
  1649. return
  1650. accountDir = baseDir + '/accounts/' + handle
  1651. if not os.path.isdir(accountDir):
  1652. return
  1653. likeFile = accountDir + '/.newLike'
  1654. if os.path.isfile(likeFile):
  1655. if '##sent##' not in open(likeFile).read():
  1656. return
  1657. likerNickname = getNicknameFromActor(actor)
  1658. likerDomain, likerPort = getDomainFromActor(actor)
  1659. if likerNickname and likerDomain:
  1660. likerHandle = likerNickname + '@' + likerDomain
  1661. else:
  1662. print('likeNotify likerHandle: ' +
  1663. str(likerNickname) + '@' + str(likerDomain))
  1664. likerHandle = actor
  1665. if likerHandle != handle:
  1666. likeStr = likerHandle + ' ' + url + '?likedBy=' + actor
  1667. prevLikeFile = accountDir + '/.prevLike'
  1668. # was there a previous like notification?
  1669. if os.path.isfile(prevLikeFile):
  1670. # is it the same as the current notification ?
  1671. with open(prevLikeFile, 'r') as fp:
  1672. prevLikeStr = fp.read()
  1673. if prevLikeStr == likeStr:
  1674. return
  1675. try:
  1676. with open(prevLikeFile, 'w+') as fp:
  1677. fp.write(likeStr)
  1678. except BaseException:
  1679. print('ERROR: unable to save previous like notification ' +
  1680. prevLikeFile)
  1681. pass
  1682. try:
  1683. with open(likeFile, 'w+') as fp:
  1684. fp.write(likeStr)
  1685. except BaseException:
  1686. print('ERROR: unable to write like notification file ' +
  1687. likeFile)
  1688. pass
  1689. def replyNotify(baseDir: str, handle: str, url: str) -> None:
  1690. """Creates a notification that a new reply has arrived
  1691. """
  1692. accountDir = baseDir + '/accounts/' + handle
  1693. if not os.path.isdir(accountDir):
  1694. return
  1695. replyFile = accountDir + '/.newReply'
  1696. if not os.path.isfile(replyFile):
  1697. with open(replyFile, 'w+') as fp:
  1698. fp.write(url)
  1699. def gitPatchNotify(baseDir: str, handle: str,
  1700. subject: str, content: str,
  1701. fromNickname: str, fromDomain: str) -> None:
  1702. """Creates a notification that a new git patch has arrived
  1703. """
  1704. accountDir = baseDir + '/accounts/' + handle
  1705. if not os.path.isdir(accountDir):
  1706. return
  1707. patchFile = accountDir + '/.newPatch'
  1708. subject = subject.replace('[PATCH]', '').strip()
  1709. handle = '@' + fromNickname + '@' + fromDomain
  1710. with open(patchFile, 'w+') as fp:
  1711. fp.write('git ' + handle + ' ' + subject)
  1712. def groupHandle(baseDir: str, handle: str) -> bool:
  1713. """Is the given account handle a group?
  1714. """
  1715. actorFile = baseDir + '/accounts/' + handle + '.json'
  1716. if not os.path.isfile(actorFile):
  1717. return False
  1718. actorJson = loadJson(actorFile)
  1719. if not actorJson:
  1720. return False
  1721. return actorJson['type'] == 'Group'
  1722. def getGroupName(baseDir: str, handle: str) -> str:
  1723. """Returns the preferred name of a group
  1724. """
  1725. actorFile = baseDir + '/accounts/' + handle + '.json'
  1726. if not os.path.isfile(actorFile):
  1727. return False
  1728. actorJson = loadJson(actorFile)
  1729. if not actorJson:
  1730. return 'Group'
  1731. return actorJson['name']
  1732. def sendToGroupMembers(session, baseDir: str, handle: str, port: int,
  1733. postJsonObject: {},
  1734. httpPrefix: str, federationList: [],
  1735. sendThreads: [], postLog: [], cachedWebfingers: {},
  1736. personCache: {}, debug: bool) -> None:
  1737. """When a post arrives for a group send it out to the group members
  1738. """
  1739. followersFile = baseDir + '/accounts/' + handle + '/followers.txt'
  1740. if not os.path.isfile(followersFile):
  1741. return
  1742. if not postJsonObject.get('object'):
  1743. return
  1744. nickname = handle.split('@')[0]
  1745. # groupname = getGroupName(baseDir, handle)
  1746. domain = handle.split('@')[1]
  1747. domainFull = domain
  1748. if ':' not in domain:
  1749. if port:
  1750. if port != 80 and port != 443:
  1751. domain = domain + ':' + str(port)
  1752. # set sender
  1753. cc = ''
  1754. sendingActor = postJsonObject['actor']
  1755. sendingActorNickname = getNicknameFromActor(sendingActor)
  1756. sendingActorDomain, sendingActorPort = \
  1757. getDomainFromActor(sendingActor)
  1758. sendingActorDomainFull = sendingActorDomain
  1759. if ':' in sendingActorDomain:
  1760. if sendingActorPort:
  1761. if sendingActorPort != 80 and sendingActorPort != 443:
  1762. sendingActorDomainFull = \
  1763. sendingActorDomain + ':' + str(sendingActorPort)
  1764. senderStr = '@' + sendingActorNickname + '@' + sendingActorDomainFull
  1765. if not postJsonObject['object']['content'].startswith(senderStr):
  1766. postJsonObject['object']['content'] = \
  1767. senderStr + ' ' + postJsonObject['object']['content']
  1768. # add mention to tag list
  1769. if not postJsonObject['object']['tag']:
  1770. postJsonObject['object']['tag'] = []
  1771. # check if the mention already exists
  1772. mentionExists = False
  1773. for mention in postJsonObject['object']['tag']:
  1774. if mention['type'] == 'Mention':
  1775. if mention.get('href'):
  1776. if mention['href'] == sendingActor:
  1777. mentionExists = True
  1778. if not mentionExists:
  1779. # add the mention of the original sender
  1780. postJsonObject['object']['tag'].append({
  1781. 'href': sendingActor,
  1782. 'name': senderStr,
  1783. 'type': 'Mention'
  1784. })
  1785. postJsonObject['actor'] = \
  1786. httpPrefix + '://' + domainFull + '/users/' + nickname
  1787. postJsonObject['to'] = \
  1788. [postJsonObject['actor'] + '/followers']
  1789. postJsonObject['cc'] = [cc]
  1790. postJsonObject['object']['to'] = postJsonObject['to']
  1791. postJsonObject['object']['cc'] = [cc]
  1792. # set subject
  1793. if not postJsonObject['object'].get('summary'):
  1794. postJsonObject['object']['summary'] = 'General Discussion'
  1795. if ':' in domain:
  1796. domain = domain.split(':')[0]
  1797. with open(followersFile, 'r') as groupMembers:
  1798. for memberHandle in groupMembers:
  1799. if memberHandle != handle:
  1800. memberNickname = memberHandle.split('@')[0]
  1801. memberDomain = memberHandle.split('@')[1]
  1802. memberPort = port
  1803. if ':' in memberDomain:
  1804. memberPortStr = memberDomain.split(':')[1]
  1805. if memberPortStr.isdigit():
  1806. memberPort = int(memberPortStr)
  1807. memberDomain = memberDomain.split(':')[0]
  1808. sendSignedJson(postJsonObject, session, baseDir,
  1809. nickname, domain, port,
  1810. memberNickname, memberDomain, memberPort, cc,
  1811. httpPrefix, False, False, federationList,
  1812. sendThreads, postLog, cachedWebfingers,
  1813. personCache, debug, __version__)
  1814. def inboxUpdateCalendar(baseDir: str, handle: str, postJsonObject: {}) -> None:
  1815. """Detects whether the tag list on a post contains calendar events
  1816. and if so saves the post id to a file in the calendar directory
  1817. for the account
  1818. """
  1819. if not postJsonObject.get('actor'):
  1820. return
  1821. if not postJsonObject.get('object'):
  1822. return
  1823. if not isinstance(postJsonObject['object'], dict):
  1824. return
  1825. if not postJsonObject['object'].get('tag'):
  1826. return
  1827. if not isinstance(postJsonObject['object']['tag'], list):
  1828. return
  1829. actor = postJsonObject['actor']
  1830. actorNickname = getNicknameFromActor(actor)
  1831. actorDomain, actorPort = getDomainFromActor(actor)
  1832. handleNickname = handle.split('@')[0]
  1833. handleDomain = handle.split('@')[1]
  1834. if not receivingCalendarEvents(baseDir,
  1835. handleNickname, handleDomain,
  1836. actorNickname, actorDomain):
  1837. return
  1838. postId = \
  1839. postJsonObject['id'].replace('/activity', '').replace('/', '#')
  1840. # look for events within the tags list
  1841. for tagDict in postJsonObject['object']['tag']:
  1842. if not tagDict.get('type'):
  1843. continue
  1844. if tagDict['type'] != 'Event':
  1845. continue
  1846. if not tagDict.get('startTime'):
  1847. continue
  1848. saveEvent(baseDir, handle, postId, tagDict)
  1849. def inboxUpdateIndex(boxname: str, baseDir: str, handle: str,
  1850. destinationFilename: str, debug: bool) -> bool:
  1851. """Updates the index of received posts
  1852. The new entry is added to the top of the file
  1853. """
  1854. indexFilename = baseDir + '/accounts/' + handle + '/' + boxname + '.index'
  1855. if debug:
  1856. print('DEBUG: Updating index ' + indexFilename)
  1857. if '/' + boxname + '/' in destinationFilename:
  1858. destinationFilename = destinationFilename.split('/' + boxname + '/')[1]
  1859. # remove the path
  1860. if '/' in destinationFilename:
  1861. destinationFilename = destinationFilename.split('/')[-1]
  1862. if os.path.isfile(indexFilename):
  1863. try:
  1864. with open(indexFilename, 'r+') as indexFile:
  1865. content = indexFile.read()
  1866. indexFile.seek(0, 0)
  1867. indexFile.write(destinationFilename + '\n' + content)
  1868. return True
  1869. except Exception as e:
  1870. print('WARN: Failed to write entry to index ' + str(e))
  1871. else:
  1872. try:
  1873. indexFile = open(indexFilename, 'w+')
  1874. if indexFile:
  1875. indexFile.write(destinationFilename + '\n')
  1876. indexFile.close()
  1877. except Exception as e:
  1878. print('WARN: Failed to write initial entry to index ' + str(e))
  1879. return False
  1880. def inboxAfterCapabilities(recentPostsCache: {}, maxRecentPosts: int,
  1881. session, keyId: str, handle: str, messageJson: {},
  1882. baseDir: str, httpPrefix: str, sendThreads: [],
  1883. postLog: [], cachedWebfingers: {}, personCache: {},
  1884. queue: [], domain: str,
  1885. onionDomain: str, i2pDomain: str,
  1886. port: int, proxyType: str,
  1887. federationList: [], ocapAlways: bool, debug: bool,
  1888. acceptedCaps: [],
  1889. queueFilename: str, destinationFilename: str,
  1890. maxReplies: int, allowDeletion: bool,
  1891. maxMentions: int, maxEmoji: int, translate: {},
  1892. unitTest: bool, YTReplacementDomain: str) -> bool:
  1893. """ Anything which needs to be done after capabilities checks have passed
  1894. """
  1895. actor = keyId
  1896. if '#' in actor:
  1897. actor = keyId.split('#')[0]
  1898. isGroup = groupHandle(baseDir, handle)
  1899. if receiveLike(recentPostsCache,
  1900. session, handle, isGroup,
  1901. baseDir, httpPrefix,
  1902. domain, port,
  1903. onionDomain,
  1904. sendThreads, postLog,
  1905. cachedWebfingers,
  1906. personCache,
  1907. messageJson,
  1908. federationList,
  1909. debug):
  1910. if debug:
  1911. print('DEBUG: Like accepted from ' + actor)
  1912. return False
  1913. if receiveUndoLike(recentPostsCache,
  1914. session, handle, isGroup,
  1915. baseDir, httpPrefix,
  1916. domain, port,
  1917. sendThreads, postLog,
  1918. cachedWebfingers,
  1919. personCache,
  1920. messageJson,
  1921. federationList,
  1922. debug):
  1923. if debug:
  1924. print('DEBUG: Undo like accepted from ' + actor)
  1925. return False
  1926. if receiveBookmark(recentPostsCache,
  1927. session, handle, isGroup,
  1928. baseDir, httpPrefix,
  1929. domain, port,
  1930. sendThreads, postLog,
  1931. cachedWebfingers,
  1932. personCache,
  1933. messageJson,
  1934. federationList,
  1935. debug):
  1936. if debug:
  1937. print('DEBUG: Bookmark accepted from ' + actor)
  1938. return False
  1939. if receiveUndoBookmark(recentPostsCache,
  1940. session, handle, isGroup,
  1941. baseDir, httpPrefix,
  1942. domain, port,
  1943. sendThreads, postLog,
  1944. cachedWebfingers,
  1945. personCache,
  1946. messageJson,
  1947. federationList,
  1948. debug):
  1949. if debug:
  1950. print('DEBUG: Undo bookmark accepted from ' + actor)
  1951. return False
  1952. # labelAccusatoryPost(messageJson, translate)
  1953. if receiveAnnounce(recentPostsCache,
  1954. session, handle, isGroup,
  1955. baseDir, httpPrefix,
  1956. domain, onionDomain, port,
  1957. sendThreads, postLog,
  1958. cachedWebfingers,
  1959. personCache,
  1960. messageJson,
  1961. federationList,
  1962. debug, translate,
  1963. YTReplacementDomain):
  1964. if debug:
  1965. print('DEBUG: Announce accepted from ' + actor)
  1966. if receiveUndoAnnounce(recentPostsCache,
  1967. session, handle, isGroup,
  1968. baseDir, httpPrefix,
  1969. domain, port,
  1970. sendThreads, postLog,
  1971. cachedWebfingers,
  1972. personCache,
  1973. messageJson,
  1974. federationList,
  1975. debug):
  1976. if debug:
  1977. print('DEBUG: Undo announce accepted from ' + actor)
  1978. return False
  1979. if receiveDelete(session, handle, isGroup,
  1980. baseDir, httpPrefix,
  1981. domain, port,
  1982. sendThreads, postLog,
  1983. cachedWebfingers,
  1984. personCache,
  1985. messageJson,
  1986. federationList,
  1987. debug, allowDeletion,
  1988. recentPostsCache):
  1989. if debug:
  1990. print('DEBUG: Delete accepted from ' + actor)
  1991. return False
  1992. if debug:
  1993. print('DEBUG: object capabilities passed')
  1994. print('copy queue file from ' + queueFilename +
  1995. ' to ' + destinationFilename)
  1996. if os.path.isfile(destinationFilename):
  1997. return True
  1998. if messageJson.get('postNickname'):
  1999. postJsonObject = messageJson['post']
  2000. else:
  2001. postJsonObject = messageJson
  2002. nickname = handle.split('@')[0]
  2003. if validPostContent(baseDir, nickname, domain,
  2004. postJsonObject, maxMentions, maxEmoji):
  2005. # check for incoming git patches
  2006. if isinstance(postJsonObject['object'], dict):
  2007. if postJsonObject['object'].get('content') and \
  2008. postJsonObject['object'].get('summary') and \
  2009. postJsonObject['object'].get('attributedTo'):
  2010. attributedTo = postJsonObject['object']['attributedTo']
  2011. if isinstance(attributedTo, str):
  2012. fromNickname = getNicknameFromActor(attributedTo)
  2013. fromDomain, fromPort = getDomainFromActor(attributedTo)
  2014. if fromPort:
  2015. if fromPort != 80 and fromPort != 443:
  2016. fromDomain += ':' + str(fromPort)
  2017. if receiveGitPatch(baseDir, nickname, domain,
  2018. postJsonObject['object']['type'],
  2019. postJsonObject['object']['summary'],
  2020. postJsonObject['object']['content'],
  2021. fromNickname, fromDomain):
  2022. gitPatchNotify(baseDir, handle,
  2023. postJsonObject['object']['summary'],
  2024. postJsonObject['object']['content'],
  2025. fromNickname, fromDomain)
  2026. elif '[PATCH]' in postJsonObject['object']['content']:
  2027. print('WARN: git patch not accepted - ' +
  2028. postJsonObject['object']['summary'])
  2029. return False
  2030. # replace YouTube links, so they get less tracking data
  2031. replaceYouTube(postJsonObject, YTReplacementDomain)
  2032. # list of indexes to be updated
  2033. updateIndexList = ['inbox']
  2034. populateReplies(baseDir, httpPrefix, domain, postJsonObject,
  2035. maxReplies, debug)
  2036. # if this is a reply to a question then update the votes
  2037. questionJson = questionUpdateVotes(baseDir, nickname, domain,
  2038. postJsonObject)
  2039. if questionJson:
  2040. # Is this a question created by this instance?
  2041. idPrefix = httpPrefix + '://' + domain
  2042. if questionJson['object']['id'].startswith(idPrefix):
  2043. # if the votes on a question have changed then
  2044. # send out an update
  2045. questionJson['type'] = 'Update'
  2046. sendToFollowersThread(session, baseDir,
  2047. nickname, domain,
  2048. onionDomain, i2pDomain, port,
  2049. httpPrefix, federationList,
  2050. sendThreads, postLog,
  2051. cachedWebfingers, personCache,
  2052. postJsonObject, debug,
  2053. __version__)
  2054. if not isGroup:
  2055. # create a DM notification file if needed
  2056. postIsDM = isDM(postJsonObject)
  2057. if postIsDM:
  2058. if nickname != 'inbox':
  2059. followDMsFilename = \
  2060. baseDir + '/accounts/' + \
  2061. nickname + '@' + domain + '/.followDMs'
  2062. if os.path.isfile(followDMsFilename):
  2063. followingFilename = \
  2064. baseDir + '/accounts/' + \
  2065. nickname + '@' + domain + '/following.txt'
  2066. if not postJsonObject.get('actor'):
  2067. return False
  2068. sendingActor = postJsonObject['actor']
  2069. sendingActorNickname = \
  2070. getNicknameFromActor(sendingActor)
  2071. sendingActorDomain, sendingActorPort = \
  2072. getDomainFromActor(sendingActor)
  2073. if sendingActorNickname and sendingActorDomain:
  2074. sendH = \
  2075. sendingActorNickname + '@' + sendingActorDomain
  2076. if sendH != nickname + '@' + domain:
  2077. if sendH not in open(followingFilename).read():
  2078. print(nickname + '@' + domain +
  2079. ' cannot receive DM from ' + sendH +
  2080. ' because they do not follow them')
  2081. return False
  2082. else:
  2083. return False
  2084. # dm index will be updated
  2085. updateIndexList.append('dm')
  2086. dmNotify(baseDir, handle,
  2087. httpPrefix + '://' + domain + '/users/' +
  2088. nickname + '/dm')
  2089. # get the actor being replied to
  2090. domainFull = domain
  2091. if port:
  2092. if ':' not in domain:
  2093. if port != 80 and port != 443:
  2094. domainFull = domainFull + ':' + str(port)
  2095. actor = httpPrefix + '://' + domainFull + \
  2096. '/users/' + handle.split('@')[0]
  2097. # create a reply notification file if needed
  2098. if not postIsDM and isReply(postJsonObject, actor):
  2099. if nickname != 'inbox':
  2100. # replies index will be updated
  2101. updateIndexList.append('tlreplies')
  2102. replyNotify(baseDir, handle,
  2103. httpPrefix + '://' + domain +
  2104. '/users/' + nickname + '/tlreplies')
  2105. if isImageMedia(session, baseDir, httpPrefix,
  2106. nickname, domain, postJsonObject,
  2107. translate, YTReplacementDomain):
  2108. # media index will be updated
  2109. updateIndexList.append('tlmedia')
  2110. if isBlogPost(postJsonObject):
  2111. # blogs index will be updated
  2112. updateIndexList.append('tlblogs')
  2113. # get the avatar for a reply/announce
  2114. obtainAvatarForReplyPost(session, baseDir,
  2115. httpPrefix, domain, onionDomain,
  2116. personCache, postJsonObject, debug)
  2117. # save the post to file
  2118. if saveJson(postJsonObject, destinationFilename):
  2119. # update the indexes for different timelines
  2120. for boxname in updateIndexList:
  2121. if not inboxUpdateIndex(boxname, baseDir, handle,
  2122. destinationFilename, debug):
  2123. print('ERROR: unable to update ' + boxname + ' index')
  2124. inboxUpdateCalendar(baseDir, handle, postJsonObject)
  2125. storeHashTags(baseDir, handle.split('@')[0], postJsonObject)
  2126. if not unitTest:
  2127. if debug:
  2128. print('DEBUG: saving inbox post as html to cache')
  2129. htmlCacheStartTime = time.time()
  2130. inboxStorePostToHtmlCache(recentPostsCache, maxRecentPosts,
  2131. translate, baseDir, httpPrefix,
  2132. session, cachedWebfingers,
  2133. personCache,
  2134. handle.split('@')[0], domain, port,
  2135. postJsonObject, allowDeletion)
  2136. if debug:
  2137. timeDiff = \
  2138. str(int((time.time() - htmlCacheStartTime) * 1000))
  2139. print('DEBUG: saved inbox post as html to cache in ' +
  2140. timeDiff + ' mS')
  2141. # send the post out to group members
  2142. if isGroup:
  2143. sendToGroupMembers(session, baseDir, handle, port,
  2144. postJsonObject,
  2145. httpPrefix, federationList, sendThreads,
  2146. postLog, cachedWebfingers, personCache,
  2147. debug)
  2148. # if the post wasn't saved
  2149. if not os.path.isfile(destinationFilename):
  2150. return False
  2151. return True
  2152. def clearQueueItems(baseDir: str, queue: []) -> None:
  2153. """Clears the queue for each account
  2154. """
  2155. ctr = 0
  2156. queue.clear()
  2157. for subdir, dirs, files in os.walk(baseDir + '/accounts'):
  2158. for account in dirs:
  2159. queueDir = baseDir + '/accounts/' + account + '/queue'
  2160. if not os.path.isdir(queueDir):
  2161. continue
  2162. for queuesubdir, queuedirs, queuefiles in os.walk(queueDir):
  2163. for qfile in queuefiles:
  2164. try:
  2165. os.remove(os.path.join(queueDir, qfile))
  2166. ctr += 1
  2167. except BaseException:
  2168. pass
  2169. if ctr > 0:
  2170. print('Removed ' + str(ctr) + ' inbox queue items')
  2171. def restoreQueueItems(baseDir: str, queue: []) -> None:
  2172. """Checks the queue for each account and appends filenames
  2173. """
  2174. queue.clear()
  2175. for subdir, dirs, files in os.walk(baseDir + '/accounts'):
  2176. for account in dirs:
  2177. queueDir = baseDir + '/accounts/' + account + '/queue'
  2178. if not os.path.isdir(queueDir):
  2179. continue
  2180. for queuesubdir, queuedirs, queuefiles in os.walk(queueDir):
  2181. for qfile in queuefiles:
  2182. queue.append(os.path.join(queueDir, qfile))
  2183. if len(queue) > 0:
  2184. print('Restored ' + str(len(queue)) + ' inbox queue items')
  2185. def runInboxQueueWatchdog(projectVersion: str, httpd) -> None:
  2186. """This tries to keep the inbox thread running even if it dies
  2187. """
  2188. print('Starting inbox queue watchdog')
  2189. inboxQueueOriginal = httpd.thrInboxQueue.clone(runInboxQueue)
  2190. httpd.thrInboxQueue.start()
  2191. while True:
  2192. time.sleep(20)
  2193. if not httpd.thrInboxQueue.isAlive() or httpd.restartInboxQueue:
  2194. httpd.restartInboxQueueInProgress = True
  2195. httpd.thrInboxQueue.kill()
  2196. httpd.thrInboxQueue = inboxQueueOriginal.clone(runInboxQueue)
  2197. httpd.inboxQueue.clear()
  2198. httpd.thrInboxQueue.start()
  2199. print('Restarting inbox queue...')
  2200. httpd.restartInboxQueueInProgress = False
  2201. httpd.restartInboxQueue = False
  2202. def runInboxQueue(recentPostsCache: {}, maxRecentPosts: int,
  2203. projectVersion: str,
  2204. baseDir: str, httpPrefix: str, sendThreads: [], postLog: [],
  2205. cachedWebfingers: {}, personCache: {}, queue: [],
  2206. domain: str,
  2207. onionDomain: str, i2pDomain: str, port: int, proxyType: str,
  2208. federationList: [],
  2209. ocapAlways: bool, maxReplies: int,
  2210. domainMaxPostsPerDay: int, accountMaxPostsPerDay: int,
  2211. allowDeletion: bool, debug: bool, maxMentions: int,
  2212. maxEmoji: int, translate: {}, unitTest: bool,
  2213. YTReplacementDomain: str,
  2214. acceptedCaps=["inbox:write", "objects:read"]) -> None:
  2215. """Processes received items and moves them to the appropriate
  2216. directories
  2217. """
  2218. currSessionTime = int(time.time())
  2219. sessionLastUpdate = currSessionTime
  2220. print('Starting new session when starting inbox queue')
  2221. session = createSession(proxyType)
  2222. inboxHandle = 'inbox@' + domain
  2223. if debug:
  2224. print('DEBUG: Inbox queue running')
  2225. # if queue processing was interrupted (eg server crash)
  2226. # then this loads any outstanding items back into the queue
  2227. restoreQueueItems(baseDir, queue)
  2228. # keep track of numbers of incoming posts per day
  2229. quotasLastUpdateDaily = int(time.time())
  2230. quotasDaily = {
  2231. 'domains': {},
  2232. 'accounts': {}
  2233. }
  2234. quotasLastUpdatePerMin = int(time.time())
  2235. quotasPerMin = {
  2236. 'domains': {},
  2237. 'accounts': {}
  2238. }
  2239. heartBeatCtr = 0
  2240. queueRestoreCtr = 0
  2241. while True:
  2242. time.sleep(1)
  2243. # heartbeat to monitor whether the inbox queue is running
  2244. heartBeatCtr += 5
  2245. if heartBeatCtr >= 10:
  2246. print('>>> Heartbeat Q:' + str(len(queue)) + ' ' +
  2247. '{:%F %T}'.format(datetime.datetime.now()))
  2248. heartBeatCtr = 0
  2249. if len(queue) == 0:
  2250. # restore any remaining queue items
  2251. queueRestoreCtr += 1
  2252. if queueRestoreCtr >= 30:
  2253. queueRestoreCtr = 0
  2254. restoreQueueItems(baseDir, queue)
  2255. continue
  2256. currTime = int(time.time())
  2257. # recreate the session periodically
  2258. if not session or currTime - sessionLastUpdate > 21600:
  2259. print('Regenerating inbox queue session at 6hr interval')
  2260. session = createSession(proxyType)
  2261. if not session:
  2262. continue
  2263. sessionLastUpdate = currTime
  2264. # oldest item first
  2265. queue.sort()
  2266. queueFilename = queue[0]
  2267. if not os.path.isfile(queueFilename):
  2268. print("Queue: queue item rejected because it has no file: " +
  2269. queueFilename)
  2270. if len(queue) > 0:
  2271. queue.pop(0)
  2272. continue
  2273. print('Loading queue item ' + queueFilename)
  2274. # Load the queue json
  2275. queueJson = loadJson(queueFilename, 1)
  2276. if not queueJson:
  2277. print('Queue: runInboxQueue failed to load inbox queue item ' +
  2278. queueFilename)
  2279. # Assume that the file is probably corrupt/unreadable
  2280. if len(queue) > 0:
  2281. queue.pop(0)
  2282. # delete the queue file
  2283. if os.path.isfile(queueFilename):
  2284. try:
  2285. os.remove(queueFilename)
  2286. except BaseException:
  2287. pass
  2288. continue
  2289. # clear the daily quotas for maximum numbers of received posts
  2290. if currTime-quotasLastUpdateDaily > 60 * 60 * 24:
  2291. quotasDaily = {
  2292. 'domains': {},
  2293. 'accounts': {}
  2294. }
  2295. quotasLastUpdateDaily = currTime
  2296. # clear the per minute quotas for maximum numbers of received posts
  2297. if currTime-quotasLastUpdatePerMin > 60:
  2298. quotasPerMin = {
  2299. 'domains': {},
  2300. 'accounts': {}
  2301. }
  2302. quotasLastUpdatePerMin = currTime
  2303. # limit the number of posts which can arrive per domain per day
  2304. postDomain = queueJson['postDomain']
  2305. if postDomain:
  2306. if domainMaxPostsPerDay > 0:
  2307. if quotasDaily['domains'].get(postDomain):
  2308. if quotasDaily['domains'][postDomain] > \
  2309. domainMaxPostsPerDay:
  2310. print('Queue: Quota per day - Maximum posts for ' +
  2311. postDomain + ' reached (' +
  2312. str(domainMaxPostsPerDay) + ')')
  2313. if len(queue) > 0:
  2314. try:
  2315. os.remove(queueFilename)
  2316. except BaseException:
  2317. pass
  2318. queue.pop(0)
  2319. continue
  2320. quotasDaily['domains'][postDomain] += 1
  2321. else:
  2322. quotasDaily['domains'][postDomain] = 1
  2323. if quotasPerMin['domains'].get(postDomain):
  2324. domainMaxPostsPerMin = \
  2325. int(domainMaxPostsPerDay / (24 * 60))
  2326. if domainMaxPostsPerMin < 5:
  2327. domainMaxPostsPerMin = 5
  2328. if quotasPerMin['domains'][postDomain] > \
  2329. domainMaxPostsPerMin:
  2330. print('Queue: Quota per min - Maximum posts for ' +
  2331. postDomain + ' reached (' +
  2332. str(domainMaxPostsPerMin) + ')')
  2333. if len(queue) > 0:
  2334. try:
  2335. os.remove(queueFilename)
  2336. except BaseException:
  2337. pass
  2338. queue.pop(0)
  2339. continue
  2340. quotasPerMin['domains'][postDomain] += 1
  2341. else:
  2342. quotasPerMin['domains'][postDomain] = 1
  2343. if accountMaxPostsPerDay > 0:
  2344. postHandle = queueJson['postNickname'] + '@' + postDomain
  2345. if quotasDaily['accounts'].get(postHandle):
  2346. if quotasDaily['accounts'][postHandle] > \
  2347. accountMaxPostsPerDay:
  2348. print('Queue: Quota account posts per day -' +
  2349. ' Maximum posts for ' +
  2350. postHandle + ' reached (' +
  2351. str(accountMaxPostsPerDay) + ')')
  2352. if len(queue) > 0:
  2353. try:
  2354. os.remove(queueFilename)
  2355. except BaseException:
  2356. pass
  2357. queue.pop(0)
  2358. continue
  2359. quotasDaily['accounts'][postHandle] += 1
  2360. else:
  2361. quotasDaily['accounts'][postHandle] = 1
  2362. if quotasPerMin['accounts'].get(postHandle):
  2363. accountMaxPostsPerMin = \
  2364. int(accountMaxPostsPerDay / (24 * 60))
  2365. if accountMaxPostsPerMin < 5:
  2366. accountMaxPostsPerMin = 5
  2367. if quotasPerMin['accounts'][postHandle] > \
  2368. accountMaxPostsPerMin:
  2369. print('Queue: Quota account posts per min -' +
  2370. ' Maximum posts for ' +
  2371. postHandle + ' reached (' +
  2372. str(accountMaxPostsPerMin) + ')')
  2373. if len(queue) > 0:
  2374. try:
  2375. os.remove(queueFilename)
  2376. except BaseException:
  2377. pass
  2378. queue.pop(0)
  2379. continue
  2380. quotasPerMin['accounts'][postHandle] += 1
  2381. else:
  2382. quotasPerMin['accounts'][postHandle] = 1
  2383. if debug:
  2384. if accountMaxPostsPerDay > 0 or domainMaxPostsPerDay > 0:
  2385. pprint(quotasDaily)
  2386. print('Obtaining public key for actor ' + queueJson['actor'])
  2387. # Try a few times to obtain the public key
  2388. pubKey = None
  2389. keyId = None
  2390. for tries in range(8):
  2391. keyId = None
  2392. signatureParams = \
  2393. queueJson['httpHeaders']['signature'].split(',')
  2394. for signatureItem in signatureParams:
  2395. if signatureItem.startswith('keyId='):
  2396. if '"' in signatureItem:
  2397. keyId = signatureItem.split('"')[1]
  2398. break
  2399. if not keyId:
  2400. print('Queue: No keyId in signature: ' +
  2401. queueJson['httpHeaders']['signature'])
  2402. pubKey = None
  2403. break
  2404. pubKey = \
  2405. getPersonPubKey(baseDir, session, keyId,
  2406. personCache, debug,
  2407. projectVersion, httpPrefix,
  2408. domain, onionDomain)
  2409. if pubKey:
  2410. if debug:
  2411. print('DEBUG: public key: ' + str(pubKey))
  2412. break
  2413. if debug:
  2414. print('DEBUG: Retry ' + str(tries+1) +
  2415. ' obtaining public key for ' + keyId)
  2416. time.sleep(1)
  2417. if not pubKey:
  2418. print('Queue: public key could not be obtained from ' + keyId)
  2419. if os.path.isfile(queueFilename):
  2420. os.remove(queueFilename)
  2421. if len(queue) > 0:
  2422. queue.pop(0)
  2423. continue
  2424. # check the signature
  2425. if debug:
  2426. print('DEBUG: checking http headers')
  2427. pprint(queueJson['httpHeaders'])
  2428. if not verifyPostHeaders(httpPrefix,
  2429. pubKey,
  2430. queueJson['httpHeaders'],
  2431. queueJson['path'], False,
  2432. queueJson['digest'],
  2433. json.dumps(queueJson['post']),
  2434. debug):
  2435. print('Queue: Header signature check failed')
  2436. pprint(queueJson['httpHeaders'])
  2437. if os.path.isfile(queueFilename):
  2438. os.remove(queueFilename)
  2439. if len(queue) > 0:
  2440. queue.pop(0)
  2441. continue
  2442. if debug:
  2443. print('DEBUG: Signature check success')
  2444. # set the id to the same as the post filename
  2445. # This makes the filename and the id consistent
  2446. # if queueJson['post'].get('id'):
  2447. # queueJson['post']['id']=queueJson['id']
  2448. if receiveUndo(session,
  2449. baseDir, httpPrefix, port,
  2450. sendThreads, postLog,
  2451. cachedWebfingers,
  2452. personCache,
  2453. queueJson['post'],
  2454. federationList,
  2455. debug,
  2456. acceptedCaps=["inbox:write", "objects:read"]):
  2457. print('Queue: Undo accepted from ' + keyId)
  2458. if os.path.isfile(queueFilename):
  2459. os.remove(queueFilename)
  2460. if len(queue) > 0:
  2461. queue.pop(0)
  2462. continue
  2463. if debug:
  2464. print('DEBUG: checking for follow requests')
  2465. if receiveFollowRequest(session,
  2466. baseDir, httpPrefix, port,
  2467. sendThreads, postLog,
  2468. cachedWebfingers,
  2469. personCache,
  2470. queueJson['post'],
  2471. federationList,
  2472. debug, projectVersion,
  2473. acceptedCaps=["inbox:write",
  2474. "objects:read"]):
  2475. if os.path.isfile(queueFilename):
  2476. os.remove(queueFilename)
  2477. if len(queue) > 0:
  2478. queue.pop(0)
  2479. print('Queue: Follow activity for ' + keyId +
  2480. ' removed from queue')
  2481. continue
  2482. else:
  2483. if debug:
  2484. print('DEBUG: No follow requests')
  2485. if receiveAcceptReject(session,
  2486. baseDir, httpPrefix, domain, port,
  2487. sendThreads, postLog,
  2488. cachedWebfingers, personCache,
  2489. queueJson['post'],
  2490. federationList, debug):
  2491. print('Queue: Accept/Reject received from ' + keyId)
  2492. if os.path.isfile(queueFilename):
  2493. os.remove(queueFilename)
  2494. if len(queue) > 0:
  2495. queue.pop(0)
  2496. continue
  2497. if receiveUpdate(recentPostsCache, session,
  2498. baseDir, httpPrefix,
  2499. domain, port,
  2500. sendThreads, postLog,
  2501. cachedWebfingers,
  2502. personCache,
  2503. queueJson['post'],
  2504. federationList,
  2505. queueJson['postNickname'],
  2506. debug):
  2507. print('Queue: Update accepted from ' + keyId)
  2508. if os.path.isfile(queueFilename):
  2509. os.remove(queueFilename)
  2510. if len(queue) > 0:
  2511. queue.pop(0)
  2512. continue
  2513. # get recipients list
  2514. recipientsDict, recipientsDictFollowers = \
  2515. inboxPostRecipients(baseDir, queueJson['post'],
  2516. httpPrefix, domain, port, debug)
  2517. if len(recipientsDict.items()) == 0 and \
  2518. len(recipientsDictFollowers.items()) == 0:
  2519. print('Queue: no recipients were resolved ' +
  2520. 'for post arriving in inbox')
  2521. if os.path.isfile(queueFilename):
  2522. os.remove(queueFilename)
  2523. if len(queue) > 0:
  2524. queue.pop(0)
  2525. continue
  2526. # if there are only a small number of followers then
  2527. # process them as if they were specifically
  2528. # addresses to particular accounts
  2529. noOfFollowItems = len(recipientsDictFollowers.items())
  2530. if noOfFollowItems > 0:
  2531. # always deliver to individual inboxes
  2532. if noOfFollowItems < 999999:
  2533. if debug:
  2534. print('DEBUG: moving ' + str(noOfFollowItems) +
  2535. ' inbox posts addressed to followers')
  2536. for handle, postItem in recipientsDictFollowers.items():
  2537. recipientsDict[handle] = postItem
  2538. recipientsDictFollowers = {}
  2539. # recipientsList = [recipientsDict, recipientsDictFollowers]
  2540. if debug:
  2541. print('*************************************')
  2542. print('Resolved recipients list:')
  2543. pprint(recipientsDict)
  2544. print('Resolved followers list:')
  2545. pprint(recipientsDictFollowers)
  2546. print('*************************************')
  2547. if queueJson['post'].get('capability'):
  2548. if not isinstance(queueJson['post']['capability'], list):
  2549. print('Queue: capability on post should be a list')
  2550. if os.path.isfile(queueFilename):
  2551. os.remove(queueFilename)
  2552. if len(queue) > 0:
  2553. queue.pop(0)
  2554. continue
  2555. # Copy any posts addressed to followers into the shared inbox
  2556. # this avoid copying file multiple times to potentially many
  2557. # individual inboxes
  2558. # This obviously bypasses object capabilities and so
  2559. # any checking will needs to be handled at the time when inbox
  2560. # GET happens on individual accounts.
  2561. # See posts.py/createBoxBase
  2562. if len(recipientsDictFollowers) > 0:
  2563. sharedInboxPostFilename = \
  2564. queueJson['destination'].replace(inboxHandle, inboxHandle)
  2565. if not os.path.isfile(sharedInboxPostFilename):
  2566. saveJson(queueJson['post'], sharedInboxPostFilename)
  2567. # for posts addressed to specific accounts
  2568. for handle, capsId in recipientsDict.items():
  2569. destination = \
  2570. queueJson['destination'].replace(inboxHandle, handle)
  2571. # check that capabilities are accepted
  2572. if queueJson['post'].get('capability'):
  2573. capabilityIdList = queueJson['post']['capability']
  2574. # does the capability id list within the post
  2575. # contain the id of the recipient with this handle?
  2576. # Here the capability id begins with the handle,
  2577. # so this could also be matched separately, but it's
  2578. # probably not necessary
  2579. if capsId in capabilityIdList:
  2580. inboxAfterCapabilities(recentPostsCache,
  2581. maxRecentPosts,
  2582. session, keyId, handle,
  2583. queueJson['post'],
  2584. baseDir, httpPrefix,
  2585. sendThreads, postLog,
  2586. cachedWebfingers,
  2587. personCache, queue,
  2588. domain,
  2589. onionDomain, i2pDomain,
  2590. port, proxyType,
  2591. federationList, ocapAlways,
  2592. debug, acceptedCaps,
  2593. queueFilename, destination,
  2594. maxReplies, allowDeletion,
  2595. maxMentions, maxEmoji,
  2596. translate, unitTest,
  2597. YTReplacementDomain)
  2598. else:
  2599. print('Queue: object capabilities check has failed')
  2600. if debug:
  2601. pprint(queueJson['post'])
  2602. else:
  2603. if not ocapAlways:
  2604. inboxAfterCapabilities(recentPostsCache,
  2605. maxRecentPosts,
  2606. session, keyId, handle,
  2607. queueJson['post'],
  2608. baseDir, httpPrefix,
  2609. sendThreads, postLog,
  2610. cachedWebfingers,
  2611. personCache, queue,
  2612. domain,
  2613. onionDomain, i2pDomain,
  2614. port, proxyType,
  2615. federationList, ocapAlways,
  2616. debug, acceptedCaps,
  2617. queueFilename, destination,
  2618. maxReplies, allowDeletion,
  2619. maxMentions, maxEmoji,
  2620. translate, unitTest,
  2621. YTReplacementDomain)
  2622. if debug:
  2623. pprint(queueJson['post'])
  2624. print('No capability list within post')
  2625. print('ocapAlways: ' + str(ocapAlways))
  2626. print('Queue: Queue post accepted')
  2627. if os.path.isfile(queueFilename):
  2628. os.remove(queueFilename)
  2629. if len(queue) > 0:
  2630. queue.pop(0)