inbox.py 92 KB

12345678910111213141516171819202122232425262728293031323334353637383940414243444546474849505152535455565758596061626364656667686970717273747576777879808182838485868788899091929394959697989910010110210310410510610710810911011111211311411511611711811912012112212312412512612712812913013113213313413513613713813914014114214314414514614714814915015115215315415515615715815916016116216316416516616716816917017117217317417517617717817918018118218318418518618718818919019119219319419519619719819920020120220320420520620720820921021121221321421521621721821922022122222322422522622722822923023123223323423523623723823924024124224324424524624724824925025125225325425525625725825926026126226326426526626726826927027127227327427527627727827928028128228328428528628728828929029129229329429529629729829930030130230330430530630730830931031131231331431531631731831932032132232332432532632732832933033133233333433533633733833934034134234334434534634734834935035135235335435535635735835936036136236336436536636736836937037137237337437537637737837938038138238338438538638738838939039139239339439539639739839940040140240340440540640740840941041141241341441541641741841942042142242342442542642742842943043143243343443543643743843944044144244344444544644744844945045145245345445545645745845946046146246346446546646746846947047147247347447547647747847948048148248348448548648748848949049149249349449549649749849950050150250350450550650750850951051151251351451551651751851952052152252352452552652752852953053153253353453553653753853954054154254354454554654754854955055155255355455555655755855956056156256356456556656756856957057157257357457557657757857958058158258358458558658758858959059159259359459559659759859960060160260360460560660760860961061161261361461561661761861962062162262362462562662762862963063163263363463563663763863964064164264364464564664764864965065165265365465565665765865966066166266366466566666766866967067167267367467567667767867968068168268368468568668768868969069169269369469569669769869970070170270370470570670770870971071171271371471571671771871972072172272372472572672772872973073173273373473573673773873974074174274374474574674774874975075175275375475575675775875976076176276376476576676776876977077177277377477577677777877978078178278378478578678778878979079179279379479579679779879980080180280380480580680780880981081181281381481581681781881982082182282382482582682782882983083183283383483583683783883984084184284384484584684784884985085185285385485585685785885986086186286386486586686786886987087187287387487587687787887988088188288388488588688788888989089189289389489589689789889990090190290390490590690790890991091191291391491591691791891992092192292392492592692792892993093193293393493593693793893994094194294394494594694794894995095195295395495595695795895996096196296396496596696796896997097197297397497597697797897998098198298398498598698798898999099199299399499599699799899910001001100210031004100510061007100810091010101110121013101410151016101710181019102010211022102310241025102610271028102910301031103210331034103510361037103810391040104110421043104410451046104710481049105010511052105310541055105610571058105910601061106210631064106510661067106810691070107110721073107410751076107710781079108010811082108310841085108610871088108910901091109210931094109510961097109810991100110111021103110411051106110711081109111011111112111311141115111611171118111911201121112211231124112511261127112811291130113111321133113411351136113711381139114011411142114311441145114611471148114911501151115211531154115511561157115811591160116111621163116411651166116711681169117011711172117311741175117611771178117911801181118211831184118511861187118811891190119111921193119411951196119711981199120012011202120312041205120612071208120912101211121212131214121512161217121812191220122112221223122412251226122712281229123012311232123312341235123612371238123912401241124212431244124512461247124812491250125112521253125412551256125712581259126012611262126312641265126612671268126912701271127212731274127512761277127812791280128112821283128412851286128712881289129012911292129312941295129612971298129913001301130213031304130513061307130813091310131113121313131413151316131713181319132013211322132313241325132613271328132913301331133213331334133513361337133813391340134113421343134413451346134713481349135013511352135313541355135613571358135913601361136213631364136513661367136813691370137113721373137413751376137713781379138013811382138313841385138613871388138913901391139213931394139513961397139813991400140114021403140414051406140714081409141014111412141314141415141614171418141914201421142214231424142514261427142814291430143114321433143414351436143714381439144014411442144314441445144614471448144914501451145214531454145514561457145814591460146114621463146414651466146714681469147014711472147314741475147614771478147914801481148214831484148514861487148814891490149114921493149414951496149714981499150015011502150315041505150615071508150915101511151215131514151515161517151815191520152115221523152415251526152715281529153015311532153315341535153615371538153915401541154215431544154515461547154815491550155115521553155415551556155715581559156015611562156315641565156615671568156915701571157215731574157515761577157815791580158115821583158415851586158715881589159015911592159315941595159615971598159916001601160216031604160516061607160816091610161116121613161416151616161716181619162016211622162316241625162616271628162916301631163216331634163516361637163816391640164116421643164416451646164716481649165016511652165316541655165616571658165916601661166216631664166516661667166816691670167116721673167416751676167716781679168016811682168316841685168616871688168916901691169216931694169516961697169816991700170117021703170417051706170717081709171017111712171317141715171617171718171917201721172217231724172517261727172817291730173117321733173417351736173717381739174017411742174317441745174617471748174917501751175217531754175517561757175817591760176117621763176417651766176717681769177017711772177317741775177617771778177917801781178217831784178517861787178817891790179117921793179417951796179717981799180018011802180318041805180618071808180918101811181218131814181518161817181818191820182118221823182418251826182718281829183018311832183318341835183618371838183918401841184218431844184518461847184818491850185118521853185418551856185718581859186018611862186318641865186618671868186918701871187218731874187518761877187818791880188118821883188418851886188718881889189018911892189318941895189618971898189919001901190219031904190519061907190819091910191119121913191419151916191719181919192019211922192319241925192619271928192919301931193219331934193519361937193819391940194119421943194419451946194719481949195019511952195319541955195619571958195919601961196219631964196519661967196819691970197119721973197419751976197719781979198019811982198319841985198619871988198919901991199219931994199519961997199819992000200120022003200420052006200720082009201020112012201320142015201620172018201920202021202220232024202520262027202820292030203120322033203420352036203720382039204020412042204320442045204620472048204920502051205220532054205520562057205820592060206120622063206420652066206720682069207020712072207320742075207620772078207920802081208220832084208520862087208820892090209120922093209420952096209720982099210021012102210321042105210621072108210921102111211221132114211521162117211821192120212121222123212421252126212721282129213021312132213321342135213621372138213921402141214221432144214521462147214821492150215121522153215421552156215721582159216021612162216321642165216621672168216921702171217221732174217521762177217821792180218121822183218421852186218721882189219021912192219321942195219621972198219922002201220222032204220522062207220822092210221122122213221422152216221722182219222022212222222322242225222622272228222922302231
  1. __filename__ = "inbox.py"
  2. __author__ = "Bob Mottram"
  3. __license__ = "AGPL3+"
  4. __version__ = "1.0.0"
  5. __maintainer__ = "Bob Mottram"
  6. __email__ = "bob@freedombone.net"
  7. __status__ = "Production"
  8. import json
  9. import os
  10. import datetime
  11. import time
  12. import json
  13. import commentjson
  14. from shutil import copyfile
  15. from utils import urlPermitted
  16. from utils import createInboxQueueDir
  17. from utils import getStatusNumber
  18. from utils import getDomainFromActor
  19. from utils import getNicknameFromActor
  20. from utils import domainPermitted
  21. from utils import locatePost
  22. from utils import deletePost
  23. from utils import removeAttachment
  24. from utils import removeModerationPostFromIndex
  25. from utils import loadJson
  26. from utils import saveJson
  27. from httpsig import verifyPostHeaders
  28. from session import createSession
  29. from session import getJson
  30. from follow import receiveFollowRequest
  31. from follow import getFollowersOfActor
  32. from follow import unfollowerOfPerson
  33. from pprint import pprint
  34. from cache import getPersonFromCache
  35. from cache import storePersonInCache
  36. from acceptreject import receiveAcceptReject
  37. from capabilities import getOcapFilename
  38. from capabilities import CapablePost
  39. from capabilities import capabilitiesReceiveUpdate
  40. from like import updateLikesCollection
  41. from like import undoLikesCollectionEntry
  42. from bookmarks import updateBookmarksCollection
  43. from bookmarks import undoBookmarksCollectionEntry
  44. from blocking import isBlocked
  45. from blocking import isBlockedDomain
  46. from filters import isFiltered
  47. from announce import updateAnnounceCollection
  48. from announce import undoAnnounceCollectionEntry
  49. from httpsig import messageContentDigest
  50. from posts import downloadAnnounce
  51. from posts import isDM
  52. from posts import isReply
  53. from posts import isImageMedia
  54. from posts import sendSignedJson
  55. from webinterface import individualPostAsHtml
  56. from webinterface import getIconsDir
  57. def inboxStorePostToHtmlCache(translate: {}, \
  58. baseDir: str,httpPrefix: str, \
  59. session,cachedWebfingers: {},personCache: {}, \
  60. nickname: str,domain: str,port: int, \
  61. postJsonObject: {}, \
  62. allowDeletion: bool) -> None:
  63. """Converts the json post into html and stores it in a cache
  64. This enables the post to be quickly displayed later
  65. """
  66. pageNumber=-999
  67. showAvatarOptions=True
  68. avatarUrl=None
  69. boxName='inbox'
  70. individualPostAsHtml(getIconsDir(baseDir),translate,pageNumber, \
  71. baseDir,session,cachedWebfingers,personCache, \
  72. nickname,domain,port,postJsonObject, \
  73. avatarUrl,True,allowDeletion, \
  74. httpPrefix,__version__,boxName, \
  75. not isDM(postJsonObject), \
  76. True,True,False,True)
  77. def validInbox(baseDir: str,nickname: str,domain: str) -> bool:
  78. """Checks whether files were correctly saved to the inbox
  79. """
  80. if ':' in domain:
  81. domain=domain.split(':')[0]
  82. inboxDir=baseDir+'/accounts/'+nickname+'@'+domain+'/inbox'
  83. if not os.path.isdir(inboxDir):
  84. return True
  85. for subdir, dirs, files in os.walk(inboxDir):
  86. for f in files:
  87. filename = os.path.join(subdir, f)
  88. if not os.path.isfile(filename):
  89. print('filename: '+filename)
  90. return False
  91. if 'postNickname' in open(filename).read():
  92. print('queue file incorrectly saved to '+filename)
  93. return False
  94. return True
  95. def validInboxFilenames(baseDir: str,nickname: str,domain: str, \
  96. expectedDomain: str,expectedPort: int) -> bool:
  97. """Used by unit tests to check that the port number gets appended to
  98. domain names within saved post filenames
  99. """
  100. if ':' in domain:
  101. domain=domain.split(':')[0]
  102. inboxDir=baseDir+'/accounts/'+nickname+'@'+domain+'/inbox'
  103. if not os.path.isdir(inboxDir):
  104. return True
  105. expectedStr=expectedDomain+':'+str(expectedPort)
  106. for subdir, dirs, files in os.walk(inboxDir):
  107. for f in files:
  108. filename = os.path.join(subdir, f)
  109. if not os.path.isfile(filename):
  110. print('filename: '+filename)
  111. return False
  112. if not expectedStr in filename:
  113. print('Expected: '+expectedStr)
  114. print('Invalid filename: '+filename)
  115. return False
  116. return True
  117. def getPersonPubKey(baseDir: str,session,personUrl: str, \
  118. personCache: {},debug: bool, \
  119. projectVersion: str,httpPrefix: str,domain: str) -> str:
  120. if not personUrl:
  121. return None
  122. personUrl=personUrl.replace('#main-key','')
  123. if personUrl.endswith('/users/inbox'):
  124. if debug:
  125. print('DEBUG: Obtaining public key for shared inbox')
  126. personUrl=personUrl.replace('/users/inbox','/inbox')
  127. personJson = getPersonFromCache(baseDir,personUrl,personCache)
  128. if not personJson:
  129. if debug:
  130. print('DEBUG: Obtaining public key for '+personUrl)
  131. asHeader = {'Accept': 'application/activity+json; profile="https://www.w3.org/ns/activitystreams"'}
  132. personJson = getJson(session,personUrl,asHeader,None,projectVersion,httpPrefix,domain)
  133. if not personJson:
  134. return None
  135. pubKey=None
  136. if personJson.get('publicKey'):
  137. if personJson['publicKey'].get('publicKeyPem'):
  138. pubKey=personJson['publicKey']['publicKeyPem']
  139. else:
  140. if personJson.get('publicKeyPem'):
  141. pubKey=personJson['publicKeyPem']
  142. if not pubKey:
  143. if debug:
  144. print('DEBUG: Public key not found for '+personUrl)
  145. storePersonInCache(baseDir,personUrl,personJson,personCache)
  146. return pubKey
  147. def inboxMessageHasParams(messageJson: {}) -> bool:
  148. """Checks whether an incoming message contains expected parameters
  149. """
  150. expectedParams=['type','actor','object']
  151. for param in expectedParams:
  152. if not messageJson.get(param):
  153. return False
  154. if not messageJson.get('to'):
  155. allowedWithoutToParam=['Like','Follow','Request','Accept','Capability','Undo']
  156. if messageJson['type'] not in allowedWithoutToParam:
  157. return False
  158. return True
  159. def inboxPermittedMessage(domain: str,messageJson: {},federationList: []) -> bool:
  160. """ check that we are receiving from a permitted domain
  161. """
  162. if not messageJson.get('actor'):
  163. return False
  164. actor=messageJson['actor']
  165. # always allow the local domain
  166. if domain in actor:
  167. return True
  168. if not urlPermitted(actor,federationList,"inbox:write"):
  169. return False
  170. alwaysAllowedTypes=('Follow','Like','Delete','Announce')
  171. if messageJson['type'] not in alwaysAllowedTypes:
  172. if not messageJson.get('object'):
  173. return True
  174. if not isinstance(messageJson['object'], dict):
  175. return False
  176. if messageJson['object'].get('inReplyTo'):
  177. inReplyTo=messageJson['object']['inReplyTo']
  178. if not urlPermitted(inReplyTo,federationList,"inbox:write"):
  179. return False
  180. return True
  181. def validPublishedDate(published: str) -> bool:
  182. currTime=datetime.datetime.utcnow()
  183. pubDate=datetime.datetime.strptime(published,"%Y-%m-%dT%H:%M:%SZ")
  184. daysSincePublished = (currTime - pubTime).days
  185. if daysSincePublished>30:
  186. return False
  187. return True
  188. def savePostToInboxQueue(baseDir: str,httpPrefix: str, \
  189. nickname: str, domain: str, \
  190. postJsonObject: {}, \
  191. messageBytes: str, \
  192. httpHeaders: {}, \
  193. postPath: str,debug: bool) -> str:
  194. """Saves the give json to the inbox queue for the person
  195. keyId specifies the actor sending the post
  196. """
  197. if len(messageBytes)>10240:
  198. print('WARN: inbox message too long '+str(len(messageBytes))+' bytes')
  199. return None
  200. originalDomain=domain
  201. if ':' in domain:
  202. domain=domain.split(':')[0]
  203. # block at the ealiest stage possible, which means the data
  204. # isn't written to file
  205. postNickname=None
  206. postDomain=None
  207. actor=None
  208. if postJsonObject.get('actor'):
  209. actor=postJsonObject['actor']
  210. postNickname=getNicknameFromActor(postJsonObject['actor'])
  211. if not postNickname:
  212. print('No post Nickname in actor '+postJsonObject['actor'])
  213. return None
  214. postDomain,postPort=getDomainFromActor(postJsonObject['actor'])
  215. if not postDomain:
  216. if debug:
  217. pprint(postJsonObject)
  218. print('No post Domain in actor')
  219. return None
  220. if isBlocked(baseDir,nickname,domain,postNickname,postDomain):
  221. if debug:
  222. print('DEBUG: post from '+postNickname+' blocked')
  223. return None
  224. if postPort:
  225. if postPort!=80 and postPort!=443:
  226. if ':' not in postDomain:
  227. postDomain=postDomain+':'+str(postPort)
  228. if postJsonObject.get('object'):
  229. if isinstance(postJsonObject['object'], dict):
  230. if postJsonObject['object'].get('inReplyTo'):
  231. if isinstance(postJsonObject['object']['inReplyTo'], str):
  232. replyDomain,replyPort=getDomainFromActor(postJsonObject['object']['inReplyTo'])
  233. if isBlockedDomain(baseDir,replyDomain):
  234. print('WARN: post contains reply from '+str(actor)+' to a blocked domain: '+replyDomain)
  235. return None
  236. else:
  237. replyNickname=getNicknameFromActor(postJsonObject['object']['inReplyTo'])
  238. if replyNickname and replyDomain:
  239. if isBlocked(baseDir,nickname,domain,replyNickname,replyDomain):
  240. print('WARN: post contains reply from '+str(actor)+ \
  241. ' to a blocked account: '+replyNickname+'@'+replyDomain)
  242. return None
  243. #else:
  244. # print('WARN: post is a reply to an unidentified account: '+postJsonObject['object']['inReplyTo'])
  245. # return None
  246. if postJsonObject['object'].get('content'):
  247. if isinstance(postJsonObject['object']['content'], str):
  248. if isFiltered(baseDir,nickname,domain,postJsonObject['object']['content']):
  249. print('WARN: post was filtered out due to content')
  250. return None
  251. originalPostId=None
  252. if postJsonObject.get('id'):
  253. originalPostId=postJsonObject['id'].replace('/activity','').replace('/undo','')
  254. currTime=datetime.datetime.utcnow()
  255. postId=None
  256. if postJsonObject.get('id'):
  257. #if '/statuses/' not in postJsonObject['id']:
  258. postId=postJsonObject['id'].replace('/activity','').replace('/undo','')
  259. published=currTime.strftime("%Y-%m-%dT%H:%M:%SZ")
  260. if not postId:
  261. statusNumber,published = getStatusNumber()
  262. if actor:
  263. postId=actor+'/statuses/'+statusNumber
  264. else:
  265. postId=httpPrefix+'://'+originalDomain+'/users/'+nickname+'/statuses/'+statusNumber
  266. # NOTE: don't change postJsonObject['id'] before signature check
  267. inboxQueueDir=createInboxQueueDir(nickname,domain,baseDir)
  268. handle=nickname+'@'+domain
  269. destination=baseDir+'/accounts/'+handle+'/inbox/'+postId.replace('/','#')+'.json'
  270. #if os.path.isfile(destination):
  271. # if debug:
  272. # print(destination)
  273. # print('DEBUG: inbox item already exists')
  274. # return None
  275. filename=inboxQueueDir+'/'+postId.replace('/','#')+'.json'
  276. sharedInboxItem=False
  277. if nickname=='inbox':
  278. nickname=originalDomain
  279. sharedInboxItem=True
  280. digestStartTime=time.time()
  281. digest=messageContentDigest(messageBytes)
  282. timeDiffStr=str(int((time.time()-digestStartTime)*1000))
  283. if debug:
  284. while len(timeDiffStr)<6:
  285. timeDiffStr='0'+timeDiffStr
  286. print('DIGEST|'+timeDiffStr+'|'+filename)
  287. newQueueItem = {
  288. 'originalId': originalPostId,
  289. 'id': postId,
  290. 'actor': actor,
  291. 'nickname': nickname,
  292. 'domain': domain,
  293. 'postNickname': postNickname,
  294. 'postDomain': postDomain,
  295. 'sharedInbox': sharedInboxItem,
  296. 'published': published,
  297. 'httpHeaders': httpHeaders,
  298. 'path': postPath,
  299. 'post': postJsonObject,
  300. 'digest': digest,
  301. 'filename': filename,
  302. 'destination': destination
  303. }
  304. if debug:
  305. print('Inbox queue item created')
  306. saveJson(newQueueItem,filename)
  307. return filename
  308. def inboxCheckCapabilities(baseDir :str,nickname :str,domain :str, \
  309. actor: str,queue: [],queueJson: {}, \
  310. capabilityId: str,debug : bool) -> bool:
  311. if nickname=='inbox':
  312. return True
  313. ocapFilename= \
  314. getOcapFilename(baseDir, \
  315. queueJson['nickname'],queueJson['domain'], \
  316. actor,'accept')
  317. if not ocapFilename:
  318. return False
  319. if not os.path.isfile(ocapFilename):
  320. if debug:
  321. print('DEBUG: capabilities for '+ \
  322. actor+' do not exist')
  323. if os.path.isfile(queueFilename):
  324. os.remove(queueFilename)
  325. if len(queue)>0:
  326. queue.pop(0)
  327. return False
  328. oc=loadJson(ocapFilename)
  329. if not oc:
  330. return False
  331. if not oc.get('id'):
  332. if debug:
  333. print('DEBUG: capabilities for '+actor+' do not contain an id')
  334. if os.path.isfile(queueFilename):
  335. os.remove(queueFilename)
  336. if len(queue)>0:
  337. queue.pop(0)
  338. return False
  339. if oc['id']!=capabilityId:
  340. if debug:
  341. print('DEBUG: capability id mismatch')
  342. if os.path.isfile(queueFilename):
  343. os.remove(queueFilename)
  344. if len(queue)>0:
  345. queue.pop(0)
  346. return False
  347. if not oc.get('capability'):
  348. if debug:
  349. print('DEBUG: missing capability list')
  350. if os.path.isfile(queueFilename):
  351. os.remove(queueFilename)
  352. if len(queue)>0:
  353. queue.pop(0)
  354. return False
  355. if not CapablePost(queueJson['post'],oc['capability'],debug):
  356. if debug:
  357. print('DEBUG: insufficient capabilities to write to inbox from '+actor)
  358. if os.path.isfile(queueFilename):
  359. os.remove(queueFilename)
  360. if len(queue)>0:
  361. queue.pop(0)
  362. return False
  363. if debug:
  364. print('DEBUG: object capabilities check success')
  365. return True
  366. def inboxPostRecipientsAdd(baseDir :str,httpPrefix :str,toList :[], \
  367. recipientsDict :{}, \
  368. domainMatch: str,domain :str, \
  369. actor :str,debug: bool) -> bool:
  370. """Given a list of post recipients (toList) from 'to' or 'cc' parameters
  371. populate a recipientsDict with the handle and capabilities id for each
  372. """
  373. followerRecipients=False
  374. for recipient in toList:
  375. if not recipient:
  376. continue
  377. # is this a to a local account?
  378. if domainMatch in recipient:
  379. # get the handle for the local account
  380. nickname=recipient.split(domainMatch)[1]
  381. handle=nickname+'@'+domain
  382. if os.path.isdir(baseDir+'/accounts/'+handle):
  383. # are capabilities granted for this account to the
  384. # sender (actor) of the post?
  385. ocapFilename=baseDir+'/accounts/'+handle+'/ocap/accept/'+actor.replace('/','#')+'.json'
  386. if os.path.isfile(ocapFilename):
  387. # read the granted capabilities and obtain the id
  388. ocapJson=loadJson(ocapFilename)
  389. if ocapJson:
  390. if ocapJson.get('id'):
  391. # append with the capabilities id
  392. recipientsDict[handle]=ocapJson['id']
  393. else:
  394. recipientsDict[handle]=None
  395. else:
  396. if debug:
  397. print('DEBUG: '+ocapFilename+' not found')
  398. recipientsDict[handle]=None
  399. else:
  400. if debug:
  401. print('DEBUG: '+baseDir+'/accounts/'+handle+' does not exist')
  402. else:
  403. if debug:
  404. print('DEBUG: '+recipient+' is not local to '+domainMatch)
  405. print(str(toList))
  406. if recipient.endswith('followers'):
  407. if debug:
  408. print('DEBUG: followers detected as post recipients')
  409. followerRecipients=True
  410. return followerRecipients,recipientsDict
  411. def inboxPostRecipients(baseDir :str,postJsonObject :{}, \
  412. httpPrefix :str,domain : str,port :int, \
  413. debug :bool) -> ([],[]):
  414. """Returns dictionaries containing the recipients of the given post
  415. The shared dictionary contains followers
  416. """
  417. recipientsDict={}
  418. recipientsDictFollowers={}
  419. if not postJsonObject.get('actor'):
  420. if debug:
  421. pprint(postJsonObject)
  422. print('WARNING: inbox post has no actor')
  423. return recipientsDict,recipientsDictFollowers
  424. if ':' in domain:
  425. domain=domain.split(':')[0]
  426. domainBase=domain
  427. if port:
  428. if port!=80 and port!=443:
  429. if ':' not in domain:
  430. domain=domain+':'+str(port)
  431. domainMatch='/'+domain+'/users/'
  432. actor = postJsonObject['actor']
  433. # first get any specific people which the post is addressed to
  434. followerRecipients=False
  435. if postJsonObject.get('object'):
  436. if isinstance(postJsonObject['object'], dict):
  437. if postJsonObject['object'].get('to'):
  438. if isinstance(postJsonObject['object']['to'], list):
  439. recipientsList=postJsonObject['object']['to']
  440. else:
  441. recipientsList=[postJsonObject['object']['to']]
  442. if debug:
  443. print('DEBUG: resolving "to"')
  444. includesFollowers,recipientsDict= \
  445. inboxPostRecipientsAdd(baseDir,httpPrefix, \
  446. recipientsList, \
  447. recipientsDict, \
  448. domainMatch,domainBase, \
  449. actor,debug)
  450. if includesFollowers:
  451. followerRecipients=True
  452. else:
  453. if debug:
  454. print('DEBUG: inbox post has no "to"')
  455. if postJsonObject['object'].get('cc'):
  456. if isinstance(postJsonObject['object']['cc'], list):
  457. recipientsList=postJsonObject['object']['cc']
  458. else:
  459. recipientsList=[postJsonObject['object']['cc']]
  460. includesFollowers,recipientsDict= \
  461. inboxPostRecipientsAdd(baseDir,httpPrefix, \
  462. recipientsList, \
  463. recipientsDict, \
  464. domainMatch,domainBase, \
  465. actor,debug)
  466. if includesFollowers:
  467. followerRecipients=True
  468. else:
  469. if debug:
  470. print('DEBUG: inbox post has no cc')
  471. else:
  472. if debug:
  473. if isinstance(postJsonObject['object'], str):
  474. if '/statuses/' in postJsonObject['object']:
  475. print('DEBUG: inbox item is a link to a post')
  476. else:
  477. if '/users/' in postJsonObject['object']:
  478. print('DEBUG: inbox item is a link to an actor')
  479. if postJsonObject.get('to'):
  480. if isinstance(postJsonObject['to'], list):
  481. recipientsList=postJsonObject['to']
  482. else:
  483. recipientsList=[postJsonObject['to']]
  484. includesFollowers,recipientsDict= \
  485. inboxPostRecipientsAdd(baseDir,httpPrefix, \
  486. recipientsList, \
  487. recipientsDict, \
  488. domainMatch,domainBase, \
  489. actor,debug)
  490. if includesFollowers:
  491. followerRecipients=True
  492. if postJsonObject.get('cc'):
  493. if isinstance(postJsonObject['cc'], list):
  494. recipientsList=postJsonObject['cc']
  495. else:
  496. recipientsList=[postJsonObject['cc']]
  497. includesFollowers,recipientsDict= \
  498. inboxPostRecipientsAdd(baseDir,httpPrefix, \
  499. recipientsList, \
  500. recipientsDict, \
  501. domainMatch,domainBase, \
  502. actor,debug)
  503. if includesFollowers:
  504. followerRecipients=True
  505. if not followerRecipients:
  506. if debug:
  507. print('DEBUG: no followers were resolved')
  508. return recipientsDict,recipientsDictFollowers
  509. # now resolve the followers
  510. recipientsDictFollowers= \
  511. getFollowersOfActor(baseDir,actor,debug)
  512. return recipientsDict,recipientsDictFollowers
  513. def receiveUndoFollow(session,baseDir: str,httpPrefix: str, \
  514. port: int,messageJson: {}, \
  515. federationList: [], \
  516. debug : bool) -> bool:
  517. if not messageJson['object'].get('actor'):
  518. if debug:
  519. print('DEBUG: follow request has no actor within object')
  520. return False
  521. if '/users/' not in messageJson['object']['actor'] and \
  522. '/channel/' not in messageJson['object']['actor'] and \
  523. '/profile/' not in messageJson['object']['actor']:
  524. if debug:
  525. print('DEBUG: "users" or "profile" missing from actor within object')
  526. return False
  527. if messageJson['object']['actor'] != messageJson['actor']:
  528. if debug:
  529. print('DEBUG: actors do not match')
  530. return False
  531. nicknameFollower=getNicknameFromActor(messageJson['object']['actor'])
  532. if not nicknameFollower:
  533. print('WARN: unable to find nickname in '+messageJson['object']['actor'])
  534. return False
  535. domainFollower,portFollower=getDomainFromActor(messageJson['object']['actor'])
  536. domainFollowerFull=domainFollower
  537. if portFollower:
  538. if portFollower!=80 and portFollower!=443:
  539. if ':' not in domainFollower:
  540. domainFollowerFull=domainFollower+':'+str(portFollower)
  541. nicknameFollowing=getNicknameFromActor(messageJson['object']['object'])
  542. if not nicknameFollowing:
  543. print('WARN: unable to find nickname in '+messageJson['object']['object'])
  544. return False
  545. domainFollowing,portFollowing=getDomainFromActor(messageJson['object']['object'])
  546. domainFollowingFull=domainFollowing
  547. if portFollowing:
  548. if portFollowing!=80 and portFollowing!=443:
  549. if ':' not in domainFollowing:
  550. domainFollowingFull=domainFollowing+':'+str(portFollowing)
  551. if unfollowerOfPerson(baseDir, \
  552. nicknameFollowing,domainFollowingFull, \
  553. nicknameFollower,domainFollowerFull, \
  554. debug):
  555. if debug:
  556. print('DEBUG: Follower '+nicknameFollower+'@'+domainFollowerFull+' was removed')
  557. return True
  558. if debug:
  559. print('DEBUG: Follower '+nicknameFollower+'@'+domainFollowerFull+' was not removed')
  560. return False
  561. def receiveUndo(session,baseDir: str,httpPrefix: str, \
  562. port: int,sendThreads: [],postLog: [], \
  563. cachedWebfingers: {},personCache: {}, \
  564. messageJson: {},federationList: [], \
  565. debug : bool, \
  566. acceptedCaps=["inbox:write","objects:read"]) -> bool:
  567. """Receives an undo request within the POST section of HTTPServer
  568. """
  569. if not messageJson['type'].startswith('Undo'):
  570. return False
  571. if debug:
  572. print('DEBUG: Undo activity received')
  573. if not messageJson.get('actor'):
  574. if debug:
  575. print('DEBUG: follow request has no actor')
  576. return False
  577. if '/users/' not in messageJson['actor'] and \
  578. '/channel/' not in messageJson['actor'] and \
  579. '/profile/' not in messageJson['actor']:
  580. if debug:
  581. print('DEBUG: "users" or "profile" missing from actor')
  582. return False
  583. if not messageJson.get('object'):
  584. if debug:
  585. print('DEBUG: '+messageJson['type']+' has no object')
  586. return False
  587. if not isinstance(messageJson['object'], dict):
  588. if debug:
  589. print('DEBUG: '+messageJson['type']+' object is not a dict')
  590. return False
  591. if not messageJson['object'].get('type'):
  592. if debug:
  593. print('DEBUG: '+messageJson['type']+' has no object type')
  594. return False
  595. if not messageJson['object'].get('object'):
  596. if debug:
  597. print('DEBUG: '+messageJson['type']+' has no object within object')
  598. return False
  599. if not isinstance(messageJson['object']['object'], str):
  600. if debug:
  601. print('DEBUG: '+messageJson['type']+' object within object is not a string')
  602. return False
  603. if messageJson['object']['type']=='Follow':
  604. return receiveUndoFollow(session,baseDir,httpPrefix, \
  605. port,messageJson, \
  606. federationList, \
  607. debug)
  608. return False
  609. def personReceiveUpdate(baseDir: str, \
  610. domain: str,port: int, \
  611. updateNickname: str,updateDomain: str,updatePort: int, \
  612. personJson: {},personCache: {},debug: bool) -> bool:
  613. """Changes an actor. eg: avatar or display name change
  614. """
  615. if debug:
  616. print('DEBUG: receiving actor update for '+personJson['url'])
  617. domainFull=domain
  618. if port:
  619. if port!=80 and port!=443:
  620. domainFull=domain+':'+str(port)
  621. updateDomainFull=updateDomain
  622. if updatePort:
  623. if updatePort!=80 and updatePort!=443:
  624. updateDomainFull=updateDomain+':'+str(updatePort)
  625. actor=updateDomainFull+'/users/'+updateNickname
  626. if actor not in personJson['id']:
  627. actor=updateDomainFull+'/profile/'+updateNickname
  628. if actor not in personJson['id']:
  629. actor=updateDomainFull+'/channel/'+updateNickname
  630. if actor not in personJson['id']:
  631. if debug:
  632. print('actor: '+actor)
  633. print('id: '+personJson['id'])
  634. print('DEBUG: Actor does not match id')
  635. return False
  636. if updateDomainFull==domainFull:
  637. if debug:
  638. print('DEBUG: You can only receive actor updates for domains other than your own')
  639. return False
  640. if not personJson.get('publicKey'):
  641. if debug:
  642. print('DEBUG: actor update does not contain a public key')
  643. return False
  644. if not personJson['publicKey'].get('publicKeyPem'):
  645. if debug:
  646. print('DEBUG: actor update does not contain a public key Pem')
  647. return False
  648. actorFilename=baseDir+'/cache/actors/'+personJson['id'].replace('/','#')+'.json'
  649. # check that the public keys match.
  650. # If they don't then this may be a nefarious attempt to hack an account
  651. if personCache.get(personJson['id']):
  652. if personCache[personJson['id']]['actor']['publicKey']['publicKeyPem']!=personJson['publicKey']['publicKeyPem']:
  653. if debug:
  654. print('WARN: Public key does not match when updating actor')
  655. return False
  656. else:
  657. if os.path.isfile(actorFilename):
  658. existingPersonJson=loadJson(actorFilename)
  659. if existingPersonJson:
  660. if existingPersonJson['publicKey']['publicKeyPem']!=personJson['publicKey']['publicKeyPem']:
  661. if debug:
  662. print('WARN: Public key does not match cached actor when updating')
  663. return False
  664. # save to cache in memory
  665. storePersonInCache(baseDir,personJson['id'],personJson,personCache)
  666. # save to cache on file
  667. if saveJson(personJson,actorFilename):
  668. print('actor updated for '+personJson['id'])
  669. # remove avatar if it exists so that it will be refreshed later
  670. # when a timeline is constructed
  671. actorStr=personJson['id'].replace('/','-')
  672. avatarFilename=baseDir+'/cache/avatars/'+actorStr+'.png'
  673. if os.path.isfile(avatarFilename):
  674. os.remove(avatarFilename)
  675. else:
  676. avatarFilename=baseDir+'/cache/avatars/'+actorStr+'.jpg'
  677. if os.path.isfile(avatarFilename):
  678. os.remove(avatarFilename)
  679. else:
  680. avatarFilename=baseDir+'/cache/avatars/'+actorStr+'.gif'
  681. if os.path.isfile(avatarFilename):
  682. os.remove(avatarFilename)
  683. return True
  684. def receiveUpdate(session,baseDir: str, \
  685. httpPrefix: str,domain :str,port: int, \
  686. sendThreads: [],postLog: [],cachedWebfingers: {}, \
  687. personCache: {},messageJson: {},federationList: [], \
  688. debug : bool) -> bool:
  689. """Receives an Update activity within the POST section of HTTPServer
  690. """
  691. if messageJson['type']!='Update':
  692. return False
  693. if not messageJson.get('actor'):
  694. if debug:
  695. print('DEBUG: '+messageJson['type']+' has no actor')
  696. return False
  697. if not messageJson.get('object'):
  698. if debug:
  699. print('DEBUG: '+messageJson['type']+' has no object')
  700. return False
  701. if not isinstance(messageJson['object'], dict):
  702. if debug:
  703. print('DEBUG: '+messageJson['type']+' object is not a dict')
  704. return False
  705. if not messageJson['object'].get('type'):
  706. if debug:
  707. print('DEBUG: '+messageJson['type']+' object has no type')
  708. return False
  709. if '/users/' not in messageJson['actor'] and \
  710. '/channel/' not in messageJson['actor'] and \
  711. '/profile/' not in messageJson['actor']:
  712. if debug:
  713. print('DEBUG: "users" or "profile" missing from actor in '+messageJson['type'])
  714. return False
  715. if messageJson['object']['type']=='Person' or \
  716. messageJson['object']['type']=='Application' or \
  717. messageJson['object']['type']=='Group' or \
  718. messageJson['object']['type']=='Service':
  719. if messageJson['object'].get('url') and messageJson['object'].get('id'):
  720. print('Request to update actor: '+messageJson['actor'])
  721. updateNickname=getNicknameFromActor(messageJson['actor'])
  722. if updateNickname:
  723. updateDomain,updatePort=getDomainFromActor(messageJson['actor'])
  724. if personReceiveUpdate(baseDir, \
  725. domain,port, \
  726. updateNickname,updateDomain,updatePort, \
  727. messageJson['object'], \
  728. personCache,debug):
  729. if debug:
  730. print('DEBUG: Profile update was received for '+messageJson['object']['url'])
  731. return True
  732. if messageJson['object'].get('capability') and messageJson['object'].get('scope'):
  733. nickname=getNicknameFromActor(messageJson['object']['scope'])
  734. if nickname:
  735. domain,tempPort=getDomainFromActor(messageJson['object']['scope'])
  736. if messageJson['object']['type']=='Capability':
  737. if capabilitiesReceiveUpdate(baseDir,nickname,domain,port,
  738. messageJson['actor'], \
  739. messageJson['object']['id'], \
  740. messageJson['object']['capability'], \
  741. debug):
  742. if debug:
  743. print('DEBUG: An update was received')
  744. return True
  745. return False
  746. def receiveLike(session,handle: str,isGroup: bool,baseDir: str, \
  747. httpPrefix: str,domain :str,port: int, \
  748. sendThreads: [],postLog: [],cachedWebfingers: {}, \
  749. personCache: {},messageJson: {},federationList: [], \
  750. debug : bool) -> bool:
  751. """Receives a Like activity within the POST section of HTTPServer
  752. """
  753. if messageJson['type']!='Like':
  754. return False
  755. if not messageJson.get('actor'):
  756. if debug:
  757. print('DEBUG: '+messageJson['type']+' has no actor')
  758. return False
  759. if not messageJson.get('object'):
  760. if debug:
  761. print('DEBUG: '+messageJson['type']+' has no object')
  762. return False
  763. if not isinstance(messageJson['object'], str):
  764. if debug:
  765. print('DEBUG: '+messageJson['type']+' object is not a string')
  766. return False
  767. if not messageJson.get('to'):
  768. if debug:
  769. print('DEBUG: '+messageJson['type']+' has no "to" list')
  770. return False
  771. if '/users/' not in messageJson['actor'] and \
  772. '/channel/' not in messageJson['actor'] and \
  773. '/profile/' not in messageJson['actor']:
  774. if debug:
  775. print('DEBUG: "users" or "profile" missing from actor in '+messageJson['type'])
  776. return False
  777. if '/statuses/' not in messageJson['object']:
  778. if debug:
  779. print('DEBUG: "statuses" missing from object in '+messageJson['type'])
  780. return False
  781. if not os.path.isdir(baseDir+'/accounts/'+handle):
  782. print('DEBUG: unknown recipient of like - '+handle)
  783. # if this post in the outbox of the person?
  784. postFilename=locatePost(baseDir,handle.split('@')[0],handle.split('@')[1],messageJson['object'])
  785. if not postFilename:
  786. if debug:
  787. print('DEBUG: post not found in inbox or outbox')
  788. print(messageJson['object'])
  789. return True
  790. if debug:
  791. print('DEBUG: liked post found in inbox')
  792. updateLikesCollection(baseDir,postFilename,messageJson['object'],messageJson['actor'],domain,debug)
  793. return True
  794. def receiveUndoLike(session,handle: str,isGroup: bool,baseDir: str, \
  795. httpPrefix: str,domain :str,port: int, \
  796. sendThreads: [],postLog: [],cachedWebfingers: {}, \
  797. personCache: {},messageJson: {},federationList: [], \
  798. debug : bool) -> bool:
  799. """Receives an undo like activity within the POST section of HTTPServer
  800. """
  801. if messageJson['type']!='Undo':
  802. return False
  803. if not messageJson.get('actor'):
  804. return False
  805. if not messageJson.get('object'):
  806. return False
  807. if not isinstance(messageJson['object'], dict):
  808. return False
  809. if not messageJson['object'].get('type'):
  810. return False
  811. if messageJson['object']['type']!='Like':
  812. return False
  813. if not messageJson['object'].get('object'):
  814. if debug:
  815. print('DEBUG: '+messageJson['type']+' like has no object')
  816. return False
  817. if not isinstance(messageJson['object']['object'], str):
  818. if debug:
  819. print('DEBUG: '+messageJson['type']+' like object is not a string')
  820. return False
  821. if '/users/' not in messageJson['actor'] and \
  822. '/channel/' not in messageJson['actor'] and \
  823. '/profile/' not in messageJson['actor']:
  824. if debug:
  825. print('DEBUG: "users" or "profile" missing from actor in '+messageJson['type']+' like')
  826. return False
  827. if '/statuses/' not in messageJson['object']['object']:
  828. if debug:
  829. print('DEBUG: "statuses" missing from like object in '+messageJson['type'])
  830. return False
  831. if not os.path.isdir(baseDir+'/accounts/'+handle):
  832. print('DEBUG: unknown recipient of undo like - '+handle)
  833. # if this post in the outbox of the person?
  834. postFilename=locatePost(baseDir,handle.split('@')[0],handle.split('@')[1],messageJson['object']['object'])
  835. if not postFilename:
  836. if debug:
  837. print('DEBUG: unliked post not found in inbox or outbox')
  838. print(messageJson['object']['object'])
  839. return True
  840. if debug:
  841. print('DEBUG: liked post found in inbox. Now undoing.')
  842. undoLikesCollectionEntry(baseDir,postFilename,messageJson['object'],messageJson['actor'],domain,debug)
  843. return True
  844. def receiveBookmark(session,handle: str,isGroup: bool,baseDir: str, \
  845. httpPrefix: str,domain :str,port: int, \
  846. sendThreads: [],postLog: [],cachedWebfingers: {}, \
  847. personCache: {},messageJson: {},federationList: [], \
  848. debug : bool) -> bool:
  849. """Receives a bookmark activity within the POST section of HTTPServer
  850. """
  851. if messageJson['type']!='Bookmark':
  852. return False
  853. if not messageJson.get('actor'):
  854. if debug:
  855. print('DEBUG: '+messageJson['type']+' has no actor')
  856. return False
  857. if not messageJson.get('object'):
  858. if debug:
  859. print('DEBUG: '+messageJson['type']+' has no object')
  860. return False
  861. if not isinstance(messageJson['object'], str):
  862. if debug:
  863. print('DEBUG: '+messageJson['type']+' object is not a string')
  864. return False
  865. if not messageJson.get('to'):
  866. if debug:
  867. print('DEBUG: '+messageJson['type']+' has no "to" list')
  868. return False
  869. if '/users/' not in messageJson['actor']:
  870. if debug:
  871. print('DEBUG: "users" missing from actor in '+messageJson['type'])
  872. return False
  873. if '/statuses/' not in messageJson['object']:
  874. if debug:
  875. print('DEBUG: "statuses" missing from object in '+messageJson['type'])
  876. return False
  877. if domain not in handle.split('@')[1]:
  878. if debug:
  879. print('DEBUG: unrecognized domain '+handle)
  880. return False
  881. domainFull=domain
  882. if port:
  883. if port!=80 and port!=443:
  884. domainFull=domain+':'+str(port)
  885. nickname=handle.split('@')[0]
  886. if not messageJson['actor'].endswith(domainFull+'/users/'+nickname):
  887. if debug:
  888. print('DEBUG: bookmark actor should be the same as the handle sent to '+handle+' != '+messageJson['actor'])
  889. return False
  890. if not os.path.isdir(baseDir+'/accounts/'+handle):
  891. print('DEBUG: unknown recipient of bookmark - '+handle)
  892. # if this post in the outbox of the person?
  893. postFilename=locatePost(baseDir,nickname,domain,messageJson['object'])
  894. if not postFilename:
  895. if debug:
  896. print('DEBUG: post not found in inbox or outbox')
  897. print(messageJson['object'])
  898. return True
  899. if debug:
  900. print('DEBUG: bookmarked post was found')
  901. updateBookmarksCollection(baseDir,postFilename,messageJson['object'],messageJson['actor'],domain,debug)
  902. return True
  903. def receiveUndoBookmark(session,handle: str,isGroup: bool,baseDir: str, \
  904. httpPrefix: str,domain :str,port: int, \
  905. sendThreads: [],postLog: [],cachedWebfingers: {}, \
  906. personCache: {},messageJson: {},federationList: [], \
  907. debug : bool) -> bool:
  908. """Receives an undo bookmark activity within the POST section of HTTPServer
  909. """
  910. if messageJson['type']!='Undo':
  911. return False
  912. if not messageJson.get('actor'):
  913. return False
  914. if not messageJson.get('object'):
  915. return False
  916. if not isinstance(messageJson['object'], dict):
  917. return False
  918. if not messageJson['object'].get('type'):
  919. return False
  920. if messageJson['object']['type']!='Bookmark':
  921. return False
  922. if not messageJson['object'].get('object'):
  923. if debug:
  924. print('DEBUG: '+messageJson['type']+' like has no object')
  925. return False
  926. if not isinstance(messageJson['object']['object'], str):
  927. if debug:
  928. print('DEBUG: '+messageJson['type']+' like object is not a string')
  929. return False
  930. if '/users/' not in messageJson['actor']:
  931. if debug:
  932. print('DEBUG: "users" missing from actor in '+messageJson['type']+' like')
  933. return False
  934. if '/statuses/' not in messageJson['object']['object']:
  935. if debug:
  936. print('DEBUG: "statuses" missing from like object in '+messageJson['type'])
  937. return False
  938. domainFull=domain
  939. if port:
  940. if port!=80 and port!=443:
  941. domainFull=domain+':'+str(port)
  942. nickname=handle.split('@')[0]
  943. if domain not in handle.split('@')[1]:
  944. if debug:
  945. print('DEBUG: unrecognized bookmark domain '+handle)
  946. return False
  947. if not messageJson['actor'].endswith(domainFull+'/users/'+nickname):
  948. if debug:
  949. print('DEBUG: bookmark actor should be the same as the handle sent to '+handle+' != '+messageJson['actor'])
  950. return False
  951. if not os.path.isdir(baseDir+'/accounts/'+handle):
  952. print('DEBUG: unknown recipient of bookmark undo - '+handle)
  953. # if this post in the outbox of the person?
  954. postFilename=locatePost(baseDir,nickname,domain,messageJson['object']['object'])
  955. if not postFilename:
  956. if debug:
  957. print('DEBUG: unbookmarked post not found in inbox or outbox')
  958. print(messageJson['object']['object'])
  959. return True
  960. if debug:
  961. print('DEBUG: bookmarked post found. Now undoing.')
  962. undoBookmarksCollectionEntry(baseDir,postFilename,messageJson['object'],messageJson['actor'],domain,debug)
  963. return True
  964. def receiveDelete(session,handle: str,isGroup: bool,baseDir: str, \
  965. httpPrefix: str,domain :str,port: int, \
  966. sendThreads: [],postLog: [],cachedWebfingers: {}, \
  967. personCache: {},messageJson: {},federationList: [], \
  968. debug : bool,allowDeletion: bool) -> bool:
  969. """Receives a Delete activity within the POST section of HTTPServer
  970. """
  971. if messageJson['type']!='Delete':
  972. return False
  973. if not messageJson.get('actor'):
  974. if debug:
  975. print('DEBUG: '+messageJson['type']+' has no actor')
  976. return False
  977. if debug:
  978. print('DEBUG: Delete activity arrived')
  979. if not messageJson.get('object'):
  980. if debug:
  981. print('DEBUG: '+messageJson['type']+' has no object')
  982. return False
  983. if not isinstance(messageJson['object'], str):
  984. if debug:
  985. print('DEBUG: '+messageJson['type']+' object is not a string')
  986. return False
  987. domainFull=domain
  988. if port:
  989. if port!=80 and port!=443:
  990. if ':' not in domain:
  991. domainFull=domain+':'+str(port)
  992. deletePrefix=httpPrefix+'://'+domainFull+'/'
  993. if not allowDeletion and \
  994. (not messageJson['object'].startswith(deletePrefix) or \
  995. not messageJson['actor'].startswith(deletePrefix)):
  996. if debug:
  997. print('DEBUG: delete not permitted from other instances')
  998. return False
  999. if not messageJson.get('to'):
  1000. if debug:
  1001. print('DEBUG: '+messageJson['type']+' has no "to" list')
  1002. return False
  1003. if '/users/' not in messageJson['actor'] and \
  1004. '/channel/' not in messageJson['actor'] and \
  1005. '/profile/' not in messageJson['actor']:
  1006. if debug:
  1007. print('DEBUG: "users" or "profile" missing from actor in '+messageJson['type'])
  1008. return False
  1009. if '/statuses/' not in messageJson['object']:
  1010. if debug:
  1011. print('DEBUG: "statuses" missing from object in '+messageJson['type'])
  1012. return False
  1013. if messageJson['actor'] not in messageJson['object']:
  1014. if debug:
  1015. print('DEBUG: actor is not the owner of the post to be deleted')
  1016. if not os.path.isdir(baseDir+'/accounts/'+handle):
  1017. print('DEBUG: unknown recipient of like - '+handle)
  1018. # if this post in the outbox of the person?
  1019. messageId=messageJson['object'].replace('/activity','').replace('/undo','')
  1020. removeModerationPostFromIndex(baseDir,messageId,debug)
  1021. postFilename=locatePost(baseDir,handle.split('@')[0],handle.split('@')[1],messageId)
  1022. if not postFilename:
  1023. if debug:
  1024. print('DEBUG: delete post not found in inbox or outbox')
  1025. print(messageId)
  1026. return True
  1027. deletePost(baseDir,httpPrefix,handle.split('@')[0],handle.split('@')[1],postFilename,debug)
  1028. if debug:
  1029. print('DEBUG: post deleted - '+postFilename)
  1030. return True
  1031. def receiveAnnounce(session,handle: str,isGroup: bool,baseDir: str, \
  1032. httpPrefix: str,domain :str,port: int, \
  1033. sendThreads: [],postLog: [],cachedWebfingers: {}, \
  1034. personCache: {},messageJson: {},federationList: [], \
  1035. debug : bool) -> bool:
  1036. """Receives an announce activity within the POST section of HTTPServer
  1037. """
  1038. if messageJson['type']!='Announce':
  1039. return False
  1040. if '@' not in handle:
  1041. if debug:
  1042. print('DEBUG: bad handle '+handle)
  1043. return False
  1044. if not messageJson.get('actor'):
  1045. if debug:
  1046. print('DEBUG: '+messageJson['type']+' has no actor')
  1047. return False
  1048. if debug:
  1049. print('DEBUG: receiving announce on '+handle)
  1050. if not messageJson.get('object'):
  1051. if debug:
  1052. print('DEBUG: '+messageJson['type']+' has no object')
  1053. return False
  1054. if not isinstance(messageJson['object'], str):
  1055. if debug:
  1056. print('DEBUG: '+messageJson['type']+' object is not a string')
  1057. return False
  1058. if not messageJson.get('to'):
  1059. if debug:
  1060. print('DEBUG: '+messageJson['type']+' has no "to" list')
  1061. return False
  1062. if '/users/' not in messageJson['actor'] and \
  1063. '/channel/' not in messageJson['actor'] and \
  1064. '/profile/' not in messageJson['actor']:
  1065. if debug:
  1066. print('DEBUG: "users" or "profile" missing from actor in '+messageJson['type'])
  1067. return False
  1068. if '/users/' not in messageJson['object'] and \
  1069. '/channel/' not in messageJson['object'] and \
  1070. '/profile/' not in messageJson['object']:
  1071. if debug:
  1072. print('DEBUG: "users", "channel" or "profile" missing in '+messageJson['type'])
  1073. return False
  1074. objectDomain=messageJson['object'].replace('https://','').replace('http://','').replace('dat://','')
  1075. if '/' in objectDomain:
  1076. objectDomain=objectDomain.split('/')[0]
  1077. if isBlockedDomain(baseDir,objectDomain):
  1078. if debug:
  1079. print('DEBUG: announced domain is blocked')
  1080. return False
  1081. if not os.path.isdir(baseDir+'/accounts/'+handle):
  1082. print('DEBUG: unknown recipient of announce - '+handle)
  1083. # is this post in the outbox of the person?
  1084. nickname=handle.split('@')[0]
  1085. postFilename=locatePost(baseDir,nickname,handle.split('@')[1],messageJson['object'])
  1086. if not postFilename:
  1087. if debug:
  1088. print('DEBUG: announce post not found in inbox or outbox')
  1089. print(messageJson['object'])
  1090. return True
  1091. updateAnnounceCollection(baseDir,postFilename,messageJson['actor'],domain,debug)
  1092. if debug:
  1093. print('DEBUG: Downloading announce post '+messageJson['actor']+' -> '+messageJson['object'])
  1094. postJsonObject=downloadAnnounce(session,baseDir,httpPrefix,nickname,domain,messageJson,__version__)
  1095. if postJsonObject:
  1096. if debug:
  1097. print('DEBUG: Announce post downloaded for '+messageJson['actor']+' -> '+messageJson['object'])
  1098. # Try to obtain the actor for this person
  1099. # so that their avatar can be shown
  1100. lookupActor=None
  1101. if postJsonObject.get('attributedTo'):
  1102. lookupActor=postJsonObject['attributedTo']
  1103. else:
  1104. if postJsonObject.get('object'):
  1105. if isinstance(postJsonObject['object'], dict):
  1106. if postJsonObject['object'].get('attributedTo'):
  1107. lookupActor=postJsonObject['object']['attributedTo']
  1108. if lookupActor:
  1109. if '/users/' in lookupActor or \
  1110. '/channel/' in lookupActor or \
  1111. '/profile/' in lookupActor:
  1112. if '/statuses/' in lookupActor:
  1113. lookupActor=lookupActor.split('/statuses/')[0]
  1114. if debug:
  1115. print('DEBUG: Obtaining actor for announce post '+lookupActor)
  1116. for tries in range(6):
  1117. pubKey= \
  1118. getPersonPubKey(baseDir,session,lookupActor, \
  1119. personCache,debug, \
  1120. __version__,httpPrefix,domain)
  1121. if pubKey:
  1122. print('DEBUG: public key obtained for announce: '+lookupActor)
  1123. break
  1124. if debug:
  1125. print('DEBUG: Retry '+str(tries+1)+ \
  1126. ' obtaining actor for '+lookupActor)
  1127. time.sleep(5)
  1128. if debug:
  1129. print('DEBUG: announced/repeated post arrived in inbox')
  1130. return True
  1131. def receiveUndoAnnounce(session,handle: str,isGroup: bool,baseDir: str, \
  1132. httpPrefix: str,domain :str,port: int, \
  1133. sendThreads: [],postLog: [],cachedWebfingers: {}, \
  1134. personCache: {},messageJson: {},federationList: [], \
  1135. debug : bool) -> bool:
  1136. """Receives an undo announce activity within the POST section of HTTPServer
  1137. """
  1138. if messageJson['type']!='Undo':
  1139. return False
  1140. if not messageJson.get('actor'):
  1141. return False
  1142. if not messageJson.get('object'):
  1143. return False
  1144. if not isinstance(messageJson['object'], dict):
  1145. return False
  1146. if not messageJson['object'].get('object'):
  1147. return False
  1148. if not isinstance(messageJson['object']['object'], str):
  1149. return False
  1150. if messageJson['object']['type']!='Announce':
  1151. return False
  1152. if '/users/' not in messageJson['actor'] and \
  1153. '/channel/' not in messageJson['actor'] and \
  1154. '/profile/' not in messageJson['actor']:
  1155. if debug:
  1156. print('DEBUG: "users" or "profile" missing from actor in '+messageJson['type']+' announce')
  1157. return False
  1158. if not os.path.isdir(baseDir+'/accounts/'+handle):
  1159. print('DEBUG: unknown recipient of undo announce - '+handle)
  1160. # if this post in the outbox of the person?
  1161. postFilename=locatePost(baseDir,handle.split('@')[0],handle.split('@')[1],messageJson['object']['object'])
  1162. if not postFilename:
  1163. if debug:
  1164. print('DEBUG: undo announce post not found in inbox or outbox')
  1165. print(messageJson['object']['object'])
  1166. return True
  1167. if debug:
  1168. print('DEBUG: announced/repeated post to be undone found in inbox')
  1169. postJsonObject=loadJson(postFilename)
  1170. if postJsonObject:
  1171. if not postJsonObject.get('type'):
  1172. if postJsonObject['type']!='Announce':
  1173. if debug:
  1174. print("DEBUG: Attempt to undo something which isn't an announcement")
  1175. return False
  1176. undoAnnounceCollectionEntry(baseDir,postFilename,messageJson['actor'],domain,debug)
  1177. if os.path.isfile(postFilename):
  1178. os.remove(postFilename)
  1179. return True
  1180. def populateReplies(baseDir :str,httpPrefix :str,domain :str, \
  1181. messageJson :{},maxReplies: int,debug :bool) -> bool:
  1182. """Updates the list of replies for a post on this domain if
  1183. a reply to it arrives
  1184. """
  1185. if not messageJson.get('id'):
  1186. return False
  1187. if not messageJson.get('object'):
  1188. return False
  1189. if not isinstance(messageJson['object'], dict):
  1190. return False
  1191. if not messageJson['object'].get('inReplyTo'):
  1192. return False
  1193. if not messageJson['object'].get('to'):
  1194. return False
  1195. replyTo=messageJson['object']['inReplyTo']
  1196. if debug:
  1197. print('DEBUG: post contains a reply')
  1198. # is this a reply to a post on this domain?
  1199. if not replyTo.startswith(httpPrefix+'://'+domain+'/'):
  1200. if debug:
  1201. print('DEBUG: post is a reply to another not on this domain')
  1202. print(replyTo)
  1203. print('Expected: '+httpPrefix+'://'+domain+'/')
  1204. return False
  1205. replyToNickname=getNicknameFromActor(replyTo)
  1206. if not replyToNickname:
  1207. print('DEBUG: no nickname found for '+replyTo)
  1208. return False
  1209. replyToDomain,replyToPort=getDomainFromActor(replyTo)
  1210. if not replyToDomain:
  1211. if debug:
  1212. print('DEBUG: no domain found for '+replyTo)
  1213. return False
  1214. postFilename=locatePost(baseDir,replyToNickname,replyToDomain,replyTo)
  1215. if not postFilename:
  1216. if debug:
  1217. print('DEBUG: post may have expired - '+replyTo)
  1218. return False
  1219. # populate a text file containing the ids of replies
  1220. postRepliesFilename=postFilename.replace('.json','.replies')
  1221. messageId=messageJson['id'].replace('/activity','').replace('/undo','')
  1222. if os.path.isfile(postRepliesFilename):
  1223. numLines = sum(1 for line in open(postRepliesFilename))
  1224. if numLines>maxReplies:
  1225. return False
  1226. if messageId not in open(postRepliesFilename).read():
  1227. repliesFile=open(postRepliesFilename, "a")
  1228. repliesFile.write(messageId+'\n')
  1229. repliesFile.close()
  1230. else:
  1231. repliesFile=open(postRepliesFilename, "w")
  1232. repliesFile.write(messageId+'\n')
  1233. repliesFile.close()
  1234. return True
  1235. def estimateNumberOfMentions(content: str) -> int:
  1236. """Returns a rough estimate of the number of mentions
  1237. """
  1238. return int(content.count('@')/2)
  1239. def estimateNumberOfEmoji(content: str) -> int:
  1240. """Returns a rough estimate of the number of emoji
  1241. """
  1242. return int(content.count(':')/2)
  1243. def validPostContent(messageJson: {},maxMentions: int,maxEmoji: int) -> bool:
  1244. """Is the content of a received post valid?
  1245. Check for bad html
  1246. Check for hellthreads
  1247. Check number of tags is reasonable
  1248. """
  1249. if not messageJson.get('object'):
  1250. return True
  1251. if not isinstance(messageJson['object'], dict):
  1252. return True
  1253. if not messageJson['object'].get('content'):
  1254. return True
  1255. # check for bad html
  1256. invalidStrings=['<script>','<canvas>','<style>','</html>','</body>','<br>','<hr>']
  1257. for badStr in invalidStrings:
  1258. if badStr in messageJson['object']['content']:
  1259. if messageJson['object'].get('id'):
  1260. print('REJECT ARBITRARY HTML: '+messageJson['object']['id'])
  1261. print('REJECT ARBITRARY HTML: bad string in post - '+messageJson['object']['content'])
  1262. return False
  1263. # check (rough) number of mentions
  1264. if estimateNumberOfMentions(messageJson['object']['content'])>maxMentions:
  1265. if messageJson['object'].get('id'):
  1266. print('REJECT HELLTHREAD: '+messageJson['object']['id'])
  1267. print('REJECT HELLTHREAD: Too many mentions in post - '+messageJson['object']['content'])
  1268. return False
  1269. if estimateNumberOfEmoji(messageJson['object']['content'])>maxEmoji:
  1270. if messageJson['object'].get('id'):
  1271. print('REJECT EMOJI OVERLOAD: '+messageJson['object']['id'])
  1272. print('REJECT EMOJI OVERLOAD: Too many emoji in post - '+messageJson['object']['content'])
  1273. return False
  1274. # check number of tags
  1275. if messageJson['object'].get('tag'):
  1276. if not isinstance(messageJson['object']['tag'], list):
  1277. messageJson['object']['tag']=[]
  1278. else:
  1279. if len(messageJson['object']['tag']) > maxMentions*2:
  1280. if messageJson['object'].get('id'):
  1281. print('REJECT: '+messageJson['object']['id'])
  1282. print('REJECT: Too many tags in post - '+messageJson['object']['tag'])
  1283. return False
  1284. print('ACCEPT: post content is valid')
  1285. return True
  1286. def obtainAvatarForReplyPost(session,baseDir: str,httpPrefix: str, \
  1287. domain: str,personCache: {}, \
  1288. postJsonObject: {},debug: bool) -> None:
  1289. """Tries to obtain the actor for the person being replied to
  1290. so that their avatar can later be shown
  1291. """
  1292. if not postJsonObject.get('object'):
  1293. return
  1294. if not isinstance(postJsonObject['object'], dict):
  1295. return
  1296. if not postJsonObject['object'].get('inReplyTo'):
  1297. return
  1298. lookupActor=postJsonObject['object']['inReplyTo']
  1299. if not lookupActor:
  1300. return
  1301. if not ('/users/' in lookupActor or \
  1302. '/channel/' in lookupActor or \
  1303. '/profile/' in lookupActor):
  1304. return
  1305. if '/statuses/' in lookupActor:
  1306. lookupActor=lookupActor.split('/statuses/')[0]
  1307. if debug:
  1308. print('DEBUG: Obtaining actor for reply post '+lookupActor)
  1309. for tries in range(6):
  1310. pubKey= \
  1311. getPersonPubKey(baseDir,session,lookupActor, \
  1312. personCache,debug, \
  1313. __version__,httpPrefix,domain)
  1314. if pubKey:
  1315. print('DEBUG: public key obtained for reply: '+lookupActor)
  1316. break
  1317. if debug:
  1318. print('DEBUG: Retry '+str(tries+1)+ \
  1319. ' obtaining actor for '+lookupActor)
  1320. time.sleep(5)
  1321. def dmNotify(baseDir: str,handle: str,url: str) -> None:
  1322. """Creates a notification that a new DM has arrived
  1323. """
  1324. accountDir=baseDir+'/accounts/'+handle
  1325. if not os.path.isdir(accountDir):
  1326. return
  1327. dmFile=accountDir+'/.newDM'
  1328. if not os.path.isfile(dmFile):
  1329. with open(dmFile, 'w') as fp:
  1330. fp.write(url)
  1331. def replyNotify(baseDir: str,handle: str,url: str) -> None:
  1332. """Creates a notification that a new reply has arrived
  1333. """
  1334. accountDir=baseDir+'/accounts/'+handle
  1335. if not os.path.isdir(accountDir):
  1336. return
  1337. replyFile=accountDir+'/.newReply'
  1338. if not os.path.isfile(replyFile):
  1339. with open(replyFile, 'w') as fp:
  1340. fp.write(url)
  1341. def groupHandle(baseDir: str,handle: str) -> bool:
  1342. """Is the given account handle a group?
  1343. """
  1344. actorFile=baseDir+'/accounts/'+handle+'.json'
  1345. if not os.path.isfile(actorFile):
  1346. return False
  1347. actorJson=loadJson(actorFile)
  1348. if not actorJson:
  1349. return False
  1350. return actorJson['type']=='Group'
  1351. def getGroupName(baseDir: str,handle: str) -> str:
  1352. """Returns the preferred name of a group
  1353. """
  1354. actorFile=baseDir+'/accounts/'+handle+'.json'
  1355. if not os.path.isfile(actorFile):
  1356. return False
  1357. actorJson=loadJson(actorFile)
  1358. if not actorJson:
  1359. return 'Group'
  1360. return actorJson['name']
  1361. def sendToGroupMembers(session,baseDir: str,handle: str,port: int,postJsonObject: {}, \
  1362. httpPrefix: str,federationList: [], \
  1363. sendThreads: [],postLog: [],cachedWebfingers: {}, \
  1364. personCache: {},debug: bool) -> None:
  1365. """When a post arrives for a group send it out to the group members
  1366. """
  1367. followersFile=baseDir+'/accounts/'+handle+'/followers.txt'
  1368. if not os.path.isfile(followersFile):
  1369. return
  1370. if not postJsonObject.get('object'):
  1371. return
  1372. nickname=handle.split('@')[0]
  1373. groupname=getGroupName(baseDir,handle)
  1374. domain=handle.split('@')[1]
  1375. domainFull=domain
  1376. if ':' not in domain:
  1377. if port:
  1378. if port!=80 and port !=443:
  1379. domain=domain+':'+str(port)
  1380. # set sender
  1381. cc=''
  1382. sendingActor=postJsonObject['actor']
  1383. sendingActorNickname=getNicknameFromActor(sendingActor)
  1384. sendingActorDomain,sendingActorPort=getDomainFromActor(sendingActor)
  1385. sendingActorDomainFull=sendingActorDomain
  1386. if ':' in sendingActorDomain:
  1387. if sendingActorPort:
  1388. if sendingActorPort!=80 and sendingActorPort!=443:
  1389. sendingActorDomainFull=sendingActorDomain+':'+str(sendingActorPort)
  1390. senderStr='@'+sendingActorNickname+'@'+sendingActorDomainFull
  1391. if not postJsonObject['object']['content'].startswith(senderStr):
  1392. postJsonObject['object']['content']=senderStr+' '+postJsonObject['object']['content']
  1393. # add mention to tag list
  1394. if not postJsonObject['object']['tag']:
  1395. postJsonObject['object']['tag']=[]
  1396. # check if the mention already exists
  1397. mentionExists=False
  1398. for mention in postJsonObject['object']['tag']:
  1399. if mention['type']=='Mention':
  1400. if mention.get('href'):
  1401. if mention['href']==sendingActor:
  1402. mentionExists=True
  1403. if not mentionExists:
  1404. # add the mention of the original sender
  1405. postJsonObject['object']['tag'].append({
  1406. 'href': sendingActor,
  1407. 'name': senderStr,
  1408. 'type': 'Mention'
  1409. })
  1410. postJsonObject['actor']=httpPrefix+'://'+domainFull+'/users/'+nickname
  1411. postJsonObject['to']=[httpPrefix+'://'+domainFull+'/users/'+nickname+'/followers']
  1412. postJsonObject['cc']=[cc]
  1413. postJsonObject['object']['to']=postJsonObject['to']
  1414. postJsonObject['object']['cc']=[cc]
  1415. # set subject
  1416. if not postJsonObject['object'].get('summary'):
  1417. postJsonObject['object']['summary']='General Discussion'
  1418. if ':' in domain:
  1419. domain=domain.split(':')[0]
  1420. with open(followersFile, 'r') as groupMembers:
  1421. for memberHandle in groupMembers:
  1422. if memberHandle!=handle:
  1423. memberNickname=memberHandle.split('@')[0]
  1424. memberDomain=memberHandle.split('@')[1]
  1425. memberPort=port
  1426. if ':' in memberDomain:
  1427. memberPortStr=memberDomain.split(':')[1]
  1428. if memberPortStr.isdigit():
  1429. memberPort=int(memberPortStr)
  1430. memberDomain=memberDomain.split(':')[0]
  1431. sendSignedJson(postJsonObject,session,baseDir, \
  1432. nickname,domain,port, \
  1433. memberNickname,memberDomain,memberPort,cc, \
  1434. httpPrefix,False,False,federationList, \
  1435. sendThreads,postLog,cachedWebfingers, \
  1436. personCache,debug,projectVersion)
  1437. def inboxUpdateCalendar(baseDir: str,handle: str,postJsonObject: {}) -> None:
  1438. """Detects whether the tag list on a post contains calendar events
  1439. and if so saves the post id to a file in the calendar directory
  1440. for the account
  1441. """
  1442. if not postJsonObject.get('object'):
  1443. return
  1444. if not isinstance(postJsonObject['object'], dict):
  1445. return
  1446. if not postJsonObject['object'].get('tag'):
  1447. return
  1448. if not isinstance(postJsonObject['object']['tag'], list):
  1449. return
  1450. calendarPath=baseDir+'/accounts/'+handle+'/calendar'
  1451. if not os.path.isdir(calendarPath):
  1452. os.mkdir(calendarPath)
  1453. for tagDict in postJsonObject['object']['tag']:
  1454. if tagDict['type']!='Event':
  1455. continue
  1456. if not tagDict.get('startTime'):
  1457. continue
  1458. # get the year and month from the event
  1459. eventTime=datetime.datetime.strptime(tagDict['startTime'],"%Y-%m-%dT%H:%M:%S%z")
  1460. eventYear=int(eventTime.strftime("%Y"))
  1461. eventMonthNumber=int(eventTime.strftime("%m"))
  1462. eventDayOfMonth=int(eventTime.strftime("%d"))
  1463. if not os.path.isdir(calendarPath+'/'+str(eventYear)):
  1464. os.mkdir(calendarPath+'/'+str(eventYear))
  1465. calendarFilename=calendarPath+'/'+str(eventYear)+'/'+str(eventMonthNumber)+'.txt'
  1466. postId=postJsonObject['id'].replace('/activity','').replace('/','#')
  1467. if os.path.isfile(calendarFilename):
  1468. if postId in open(calendarFilename).read():
  1469. return
  1470. calendarFile=open(calendarFilename,'a+')
  1471. if calendarFile:
  1472. calendarFile.write(postId+'\n')
  1473. calendarFile.close()
  1474. calendarNotificationFilename=baseDir+'/accounts/'+handle+'/.newCalendar'
  1475. calendarNotificationFile=open(calendarNotificationFilename,'w')
  1476. if calendarNotificationFile:
  1477. calendarNotificationFile.write('/calendar?year='+str(eventYear)+'?month='+str(eventMonthNumber)+'?day='+str(eventDayOfMonth))
  1478. calendarNotificationFile.close()
  1479. def inboxUpdateIndex(boxname: str,baseDir: str,handle: str,destinationFilename: str,debug: bool) -> bool:
  1480. """Updates the index of received posts
  1481. The new entry is added to the top of the file
  1482. """
  1483. indexFilename=baseDir+'/accounts/'+handle+'/'+boxname+'.index'
  1484. if debug:
  1485. print('DEBUG: Updating index '+indexFilename)
  1486. if '/'+boxname+'/' in destinationFilename:
  1487. destinationFilename=destinationFilename.split('/'+boxname+'/')[1]
  1488. # remove the path
  1489. if '/' in destinationFilename:
  1490. destinationFilename=destinationFilename.split('/')[-1]
  1491. if os.path.isfile(indexFilename):
  1492. try:
  1493. with open(indexFilename, 'r+') as indexFile:
  1494. content = indexFile.read()
  1495. indexFile.seek(0, 0)
  1496. indexFile.write(destinationFilename+'\n'+content)
  1497. return True
  1498. except Exception as e:
  1499. print('WARN: Failed to write entry to index '+str(e))
  1500. else:
  1501. try:
  1502. indexFile=open(indexFilename,'w+')
  1503. if indexFile:
  1504. indexFile.write(destinationFilename+'\n')
  1505. indexFile.close()
  1506. except Exception as e:
  1507. print('WARN: Failed to write initial entry to index '+str(e))
  1508. return False
  1509. def inboxAfterCapabilities(session,keyId: str,handle: str,messageJson: {}, \
  1510. baseDir: str,httpPrefix: str,sendThreads: [], \
  1511. postLog: [],cachedWebfingers: {},personCache: {}, \
  1512. queue: [],domain: str,port: int,useTor: bool, \
  1513. federationList: [],ocapAlways: bool,debug: bool, \
  1514. acceptedCaps: [], \
  1515. queueFilename :str,destinationFilename :str, \
  1516. maxReplies: int,allowDeletion: bool, \
  1517. maxMentions: int,maxEmoji: int,translate: {}, \
  1518. unitTest: bool) -> bool:
  1519. """ Anything which needs to be done after capabilities checks have passed
  1520. """
  1521. actor=keyId
  1522. if '#' in actor:
  1523. actor=keyId.split('#')[0]
  1524. isGroup=groupHandle(baseDir,handle)
  1525. if receiveLike(session,handle,isGroup, \
  1526. baseDir,httpPrefix, \
  1527. domain,port, \
  1528. sendThreads,postLog, \
  1529. cachedWebfingers, \
  1530. personCache, \
  1531. messageJson, \
  1532. federationList, \
  1533. debug):
  1534. if debug:
  1535. print('DEBUG: Like accepted from '+actor)
  1536. return False
  1537. if receiveUndoLike(session,handle,isGroup, \
  1538. baseDir,httpPrefix, \
  1539. domain,port, \
  1540. sendThreads,postLog, \
  1541. cachedWebfingers, \
  1542. personCache, \
  1543. messageJson, \
  1544. federationList, \
  1545. debug):
  1546. if debug:
  1547. print('DEBUG: Undo like accepted from '+actor)
  1548. return False
  1549. if receiveBookmark(session,handle,isGroup, \
  1550. baseDir,httpPrefix, \
  1551. domain,port, \
  1552. sendThreads,postLog, \
  1553. cachedWebfingers, \
  1554. personCache, \
  1555. messageJson, \
  1556. federationList, \
  1557. debug):
  1558. if debug:
  1559. print('DEBUG: Bookmark accepted from '+actor)
  1560. return False
  1561. if receiveUndoBookmark(session,handle,isGroup, \
  1562. baseDir,httpPrefix, \
  1563. domain,port, \
  1564. sendThreads,postLog, \
  1565. cachedWebfingers, \
  1566. personCache, \
  1567. messageJson, \
  1568. federationList, \
  1569. debug):
  1570. if debug:
  1571. print('DEBUG: Undo bookmark accepted from '+actor)
  1572. return False
  1573. if receiveAnnounce(session,handle,isGroup, \
  1574. baseDir,httpPrefix, \
  1575. domain,port, \
  1576. sendThreads,postLog, \
  1577. cachedWebfingers, \
  1578. personCache, \
  1579. messageJson, \
  1580. federationList, \
  1581. debug):
  1582. if debug:
  1583. print('DEBUG: Announce accepted from '+actor)
  1584. if receiveUndoAnnounce(session,handle,isGroup, \
  1585. baseDir,httpPrefix, \
  1586. domain,port, \
  1587. sendThreads,postLog, \
  1588. cachedWebfingers, \
  1589. personCache, \
  1590. messageJson, \
  1591. federationList, \
  1592. debug):
  1593. if debug:
  1594. print('DEBUG: Undo announce accepted from '+actor)
  1595. return False
  1596. if receiveDelete(session,handle,isGroup, \
  1597. baseDir,httpPrefix, \
  1598. domain,port, \
  1599. sendThreads,postLog, \
  1600. cachedWebfingers, \
  1601. personCache, \
  1602. messageJson, \
  1603. federationList, \
  1604. debug,allowDeletion):
  1605. if debug:
  1606. print('DEBUG: Delete accepted from '+actor)
  1607. return False
  1608. if debug:
  1609. print('DEBUG: object capabilities passed')
  1610. print('copy queue file from '+queueFilename+' to '+destinationFilename)
  1611. if os.path.isfile(destinationFilename):
  1612. return True
  1613. if messageJson.get('postNickname'):
  1614. postJsonObject=messageJson['post']
  1615. else:
  1616. postJsonObject=messageJson
  1617. if validPostContent(postJsonObject,maxMentions,maxEmoji):
  1618. # list of indexes to be updated
  1619. updateIndexList=['inbox']
  1620. populateReplies(baseDir,httpPrefix,domain,messageJson,maxReplies,debug)
  1621. if not isGroup:
  1622. # create a DM notification file if needed
  1623. if isDM(postJsonObject):
  1624. nickname=handle.split('@')[0]
  1625. if nickname!='inbox':
  1626. followDMsFilename=baseDir+'/accounts/'+nickname+'@'+domain+'/.followDMs'
  1627. if os.path.isfile(followDMsFilename):
  1628. followingFilename=baseDir+'/accounts/'+nickname+'@'+domain+'/following.txt'
  1629. if not postJsonObject.get('actor'):
  1630. return False
  1631. sendingActor=postJsonObject['actor']
  1632. sendingActorNickname=getNicknameFromActor(sendingActor)
  1633. sendingActorDomain,sendingActorPort=getDomainFromActor(sendingActor)
  1634. if sendingActorNickname and sendingActorDomain:
  1635. if sendingActorNickname+'@'+sendingActorDomain != nickname+'@'+domain:
  1636. if sendingActorNickname+'@'+sendingActorDomain not in open(followingFilename).read():
  1637. print(nickname+'@'+domain+' cannot receive DM from '+sendingActorNickname+'@'+sendingActorDomain+' because they do not follow them')
  1638. return False
  1639. else:
  1640. return False
  1641. # dm index will be updated
  1642. updateIndexList.append('dm')
  1643. dmNotify(baseDir,handle,httpPrefix+'://'+domain+'/users/'+nickname+'/dm')
  1644. # get the actor being replied to
  1645. domainFull=domain
  1646. if port:
  1647. if ':' not in domain:
  1648. if port!=80 and port!=443:
  1649. domainFull=domainFull+':'+str(port)
  1650. actor=httpPrefix+'://'+domainFull+'/users/'+handle.split('@')[0]
  1651. # create a reply notification file if needed
  1652. nickname=handle.split('@')[0]
  1653. if isReply(postJsonObject,actor):
  1654. if nickname!='inbox':
  1655. # replies index will be updated
  1656. updateIndexList.append('tlreplies')
  1657. replyNotify(baseDir,handle,httpPrefix+'://'+domain+'/users/'+nickname+'/tlreplies')
  1658. if isImageMedia(session,baseDir,httpPrefix,nickname,domain,postJsonObject):
  1659. # media index will be updated
  1660. updateIndexList.append('tlmedia')
  1661. # get the avatar for a reply/announce
  1662. obtainAvatarForReplyPost(session,baseDir,httpPrefix,domain,personCache,postJsonObject,debug)
  1663. # save the post to file
  1664. if saveJson(postJsonObject,destinationFilename):
  1665. # update the indexes for different timelines
  1666. for boxname in updateIndexList:
  1667. if not inboxUpdateIndex(boxname,baseDir,handle,destinationFilename,debug):
  1668. print('ERROR: unable to update '+boxname+' index')
  1669. inboxUpdateCalendar(baseDir,handle,postJsonObject)
  1670. if not unitTest:
  1671. if debug:
  1672. print('DEBUG: saving inbox post as html to cache')
  1673. htmlCacheStartTime=time.time()
  1674. inboxStorePostToHtmlCache(translate,baseDir,httpPrefix, \
  1675. session,cachedWebfingers,personCache, \
  1676. handle.split('@')[0],domain,port, \
  1677. postJsonObject,allowDeletion)
  1678. if debug:
  1679. timeDiff=str(int((time.time()-htmlCacheStartTime)*1000))
  1680. print('DEBUG: saved inbox post as html to cache in '+timeDiff+' mS')
  1681. # send the post out to group members
  1682. if isGroup:
  1683. sendToGroupMembers(session,baseDir,handle,port,postJsonObject, \
  1684. httpPrefix,federationList,sendThreads, \
  1685. postLog,cachedWebfingers,personCache,debug)
  1686. # if the post wasn't saved
  1687. if not os.path.isfile(destinationFilename):
  1688. return False
  1689. return True
  1690. def restoreQueueItems(baseDir: str,queue: []) -> None:
  1691. """Checks the queue for each account and appends filenames
  1692. """
  1693. queue.clear()
  1694. for subdir,dirs,files in os.walk(baseDir+'/accounts'):
  1695. for account in dirs:
  1696. queueDir=baseDir+'/accounts/'+account+'/queue'
  1697. if os.path.isdir(queueDir):
  1698. for queuesubdir,queuedirs,queuefiles in os.walk(queueDir):
  1699. for qfile in queuefiles:
  1700. queue.append(os.path.join(queueDir, qfile))
  1701. if len(queue)>0:
  1702. print('Restored '+str(len(queue))+' inbox queue items')
  1703. def runInboxQueueWatchdog(projectVersion: str,httpd) -> None:
  1704. """This tries to keep the inbox thread running even if it dies
  1705. """
  1706. print('Starting inbox queue watchdog')
  1707. inboxQueueOriginal=httpd.thrInboxQueue.clone(runInboxQueue)
  1708. #httpd.thrInboxQueue=inboxQueueOriginal
  1709. httpd.thrInboxQueue.start()
  1710. while True:
  1711. time.sleep(20)
  1712. if not httpd.thrInboxQueue.isAlive():
  1713. httpd.thrInboxQueue.kill()
  1714. httpd.thrInboxQueue=inboxQueueOriginal.clone(runInboxQueue)
  1715. httpd.thrInboxQueue.start()
  1716. print('Restarting inbox queue...')
  1717. def runInboxQueue(projectVersion: str, \
  1718. baseDir: str,httpPrefix: str,sendThreads: [],postLog: [], \
  1719. cachedWebfingers: {},personCache: {},queue: [], \
  1720. domain: str,port: int,useTor: bool,federationList: [], \
  1721. ocapAlways: bool,maxReplies: int, \
  1722. domainMaxPostsPerDay: int,accountMaxPostsPerDay: int, \
  1723. allowDeletion: bool,debug: bool,maxMentions: int, \
  1724. maxEmoji: int,translate: {},unitTest: bool, \
  1725. acceptedCaps=["inbox:write","objects:read"]) -> None:
  1726. """Processes received items and moves them to
  1727. the appropriate directories
  1728. """
  1729. currSessionTime=int(time.time())
  1730. sessionLastUpdate=currSessionTime
  1731. session=createSession(useTor)
  1732. inboxHandle='inbox@'+domain
  1733. if debug:
  1734. print('DEBUG: Inbox queue running')
  1735. # if queue processing was interrupted (eg server crash)
  1736. # then this loads any outstanding items back into the queue
  1737. restoreQueueItems(baseDir,queue)
  1738. # keep track of numbers of incoming posts per unit of time
  1739. quotasLastUpdate=int(time.time())
  1740. quotas={
  1741. 'domains': {},
  1742. 'accounts': {}
  1743. }
  1744. # keep track of the number of queue item read failures
  1745. # so that if a file is corrupt then it will eventually
  1746. # be ignored rather than endlessly retried
  1747. itemReadFailed=0
  1748. heartBeatCtr=0
  1749. queueRestoreCtr=0
  1750. while True:
  1751. time.sleep(5)
  1752. # heartbeat to monitor whether the inbox queue is running
  1753. heartBeatCtr+=5
  1754. if heartBeatCtr>=10:
  1755. print('>>> Heartbeat Q:{:d} {:%F %T}'.format(len(queue), datetime.datetime.now()))
  1756. heartBeatCtr=0
  1757. if len(queue)==0:
  1758. # restore any remaining queue items
  1759. queueRestoreCtr+=1
  1760. if queueRestoreCtr>=30:
  1761. queueRestoreCtr=0
  1762. restoreQueueItems(baseDir,queue)
  1763. else:
  1764. currTime=int(time.time())
  1765. # recreate the session periodically
  1766. if not session or currTime-sessionLastUpdate>1200:
  1767. print('Creating inbox session')
  1768. session=createSession(useTor)
  1769. sessionLastUpdate=currTime
  1770. # oldest item first
  1771. queue.sort()
  1772. queueFilename=queue[0]
  1773. if not os.path.isfile(queueFilename):
  1774. if debug:
  1775. print("DEBUG: queue item rejected because it has no file: "+queueFilename)
  1776. if len(queue)>0:
  1777. queue.pop(0)
  1778. continue
  1779. print('Loading queue item '+queueFilename)
  1780. # Load the queue json
  1781. try:
  1782. with open(queueFilename, 'r') as fp:
  1783. queueJson=commentjson.load(fp)
  1784. except:
  1785. itemReadFailed+=1
  1786. print('WARN: commentjson exception runInboxQueue')
  1787. print('WARN: Failed to load inbox queue item '+queueFilename+' (try '+str(itemReadFailed)+')')
  1788. if itemReadFailed>4:
  1789. # After a few tries we can assume that the file
  1790. # is probably corrupt/unreadable
  1791. if len(queue)>0:
  1792. queue.pop(0)
  1793. itemReadFailed=0
  1794. # delete the queue file
  1795. if os.path.isfile(queueFilename):
  1796. os.remove(queueFilename)
  1797. continue
  1798. itemReadFailed=0
  1799. # clear the daily quotas for maximum numbers of received posts
  1800. if currTime-quotasLastUpdate>60*60*24:
  1801. quotas={
  1802. 'domains': {},
  1803. 'accounts': {}
  1804. }
  1805. quotasLastUpdate=currTime
  1806. # limit the number of posts which can arrive per domain per day
  1807. postDomain=queueJson['postDomain']
  1808. if postDomain:
  1809. if domainMaxPostsPerDay>0:
  1810. if quotas['domains'].get(postDomain):
  1811. if quotas['domains'][postDomain]>domainMaxPostsPerDay:
  1812. if debug:
  1813. print('DEBUG: Maximum posts for '+postDomain+' reached')
  1814. if len(queue)>0:
  1815. queue.pop(0)
  1816. continue
  1817. quotas['domains'][postDomain]+=1
  1818. else:
  1819. quotas['domains'][postDomain]=1
  1820. if accountMaxPostsPerDay>0:
  1821. postHandle=queueJson['postNickname']+'@'+postDomain
  1822. if quotas['accounts'].get(postHandle):
  1823. if quotas['accounts'][postHandle]>accountMaxPostsPerDay:
  1824. if debug:
  1825. print('DEBUG: Maximum posts for '+postHandle+' reached')
  1826. if len(queue)>0:
  1827. queue.pop(0)
  1828. continue
  1829. quotas['accounts'][postHandle]+=1
  1830. else:
  1831. quotas['accounts'][postHandle]=1
  1832. if debug:
  1833. if accountMaxPostsPerDay>0 or domainMaxPostsPerDay>0:
  1834. pprint(quotas)
  1835. print('Obtaining public key for actor '+queueJson['actor'])
  1836. # Try a few times to obtain the public key
  1837. pubKey=None
  1838. keyId=None
  1839. for tries in range(8):
  1840. keyId=None
  1841. signatureParams=queueJson['httpHeaders']['signature'].split(',')
  1842. for signatureItem in signatureParams:
  1843. if signatureItem.startswith('keyId='):
  1844. if '"' in signatureItem:
  1845. keyId=signatureItem.split('"')[1]
  1846. break
  1847. if not keyId:
  1848. if debug:
  1849. print('DEBUG: No keyId in signature: '+ \
  1850. queueJson['httpHeaders']['signature'])
  1851. if os.path.isfile(queueFilename):
  1852. os.remove(queueFilename)
  1853. if len(queue)>0:
  1854. queue.pop(0)
  1855. continue
  1856. pubKey= \
  1857. getPersonPubKey(baseDir,session,keyId, \
  1858. personCache,debug, \
  1859. projectVersion,httpPrefix,domain)
  1860. if pubKey:
  1861. if debug:
  1862. print('DEBUG: public key: '+str(pubKey))
  1863. break
  1864. if debug:
  1865. print('DEBUG: Retry '+str(tries+1)+ \
  1866. ' obtaining public key for '+keyId)
  1867. time.sleep(5)
  1868. if not pubKey:
  1869. if debug:
  1870. print('DEBUG: public key could not be obtained from '+keyId)
  1871. if os.path.isfile(queueFilename):
  1872. os.remove(queueFilename)
  1873. if len(queue)>0:
  1874. queue.pop(0)
  1875. continue
  1876. # check the signature
  1877. if debug:
  1878. print('DEBUG: checking http headers')
  1879. pprint(queueJson['httpHeaders'])
  1880. if not verifyPostHeaders(httpPrefix, \
  1881. pubKey, \
  1882. queueJson['httpHeaders'], \
  1883. queueJson['path'],False, \
  1884. queueJson['digest'], \
  1885. json.dumps(queueJson['post']), \
  1886. debug):
  1887. if debug:
  1888. print('DEBUG: Header signature check failed')
  1889. if os.path.isfile(queueFilename):
  1890. os.remove(queueFilename)
  1891. if len(queue)>0:
  1892. queue.pop(0)
  1893. continue
  1894. if debug:
  1895. print('DEBUG: Signature check success')
  1896. # set the id to the same as the post filename
  1897. # This makes the filename and the id consistent
  1898. #if queueJson['post'].get('id'):
  1899. # queueJson['post']['id']=queueJson['id']
  1900. if receiveUndo(session, \
  1901. baseDir,httpPrefix,port, \
  1902. sendThreads,postLog, \
  1903. cachedWebfingers,
  1904. personCache, \
  1905. queueJson['post'], \
  1906. federationList, \
  1907. debug, \
  1908. acceptedCaps=["inbox:write","objects:read"]):
  1909. if debug:
  1910. print('DEBUG: Undo accepted from '+keyId)
  1911. if os.path.isfile(queueFilename):
  1912. os.remove(queueFilename)
  1913. if len(queue)>0:
  1914. queue.pop(0)
  1915. continue
  1916. if debug:
  1917. print('DEBUG: checking for follow requests')
  1918. if receiveFollowRequest(session, \
  1919. baseDir,httpPrefix,port, \
  1920. sendThreads,postLog, \
  1921. cachedWebfingers,
  1922. personCache, \
  1923. queueJson['post'], \
  1924. federationList, \
  1925. debug,projectVersion, \
  1926. acceptedCaps=["inbox:write","objects:read"]):
  1927. if os.path.isfile(queueFilename):
  1928. os.remove(queueFilename)
  1929. if len(queue)>0:
  1930. queue.pop(0)
  1931. if debug:
  1932. print('DEBUG: Follow activity for '+keyId+' removed from accepted from queue')
  1933. continue
  1934. else:
  1935. if debug:
  1936. print('DEBUG: No follow requests')
  1937. if receiveAcceptReject(session, \
  1938. baseDir,httpPrefix,domain,port, \
  1939. sendThreads,postLog, \
  1940. cachedWebfingers, \
  1941. personCache, \
  1942. queueJson['post'], \
  1943. federationList, \
  1944. debug):
  1945. if debug:
  1946. print('DEBUG: Accept/Reject received from '+keyId)
  1947. if os.path.isfile(queueFilename):
  1948. os.remove(queueFilename)
  1949. if len(queue)>0:
  1950. queue.pop(0)
  1951. continue
  1952. if receiveUpdate(session, \
  1953. baseDir,httpPrefix, \
  1954. domain,port, \
  1955. sendThreads,postLog, \
  1956. cachedWebfingers, \
  1957. personCache, \
  1958. queueJson['post'], \
  1959. federationList, \
  1960. debug):
  1961. if debug:
  1962. print('DEBUG: Update accepted from '+keyId)
  1963. if os.path.isfile(queueFilename):
  1964. os.remove(queueFilename)
  1965. if len(queue)>0:
  1966. queue.pop(0)
  1967. continue
  1968. # get recipients list
  1969. recipientsDict,recipientsDictFollowers= \
  1970. inboxPostRecipients(baseDir,queueJson['post'], \
  1971. httpPrefix,domain,port,debug)
  1972. if len(recipientsDict.items())==0 and \
  1973. len(recipientsDictFollowers.items())==0:
  1974. if debug:
  1975. pprint(queueJson['post'])
  1976. print('DEBUG: no recipients were resolved for post arriving in inbox')
  1977. if os.path.isfile(queueFilename):
  1978. os.remove(queueFilename)
  1979. if len(queue)>0:
  1980. queue.pop(0)
  1981. continue
  1982. # if there are only a small number of followers then process them as if they
  1983. # were specifically addresses to particular accounts
  1984. noOfFollowItems=len(recipientsDictFollowers.items())
  1985. if noOfFollowItems>0:
  1986. if noOfFollowItems<5:
  1987. if debug:
  1988. print('DEBUG: moving '+str(noOfFollowItems)+ \
  1989. ' inbox posts addressed to followers')
  1990. for handle,postItem in recipientsDictFollowers.items():
  1991. recipientsDict[handle]=postItem
  1992. recipientsDictFollowers={}
  1993. recipientsList=[recipientsDict,recipientsDictFollowers]
  1994. if debug:
  1995. print('*************************************')
  1996. print('Resolved recipients list:')
  1997. pprint(recipientsDict)
  1998. print('Resolved followers list:')
  1999. pprint(recipientsDictFollowers)
  2000. print('*************************************')
  2001. if queueJson['post'].get('capability'):
  2002. if not isinstance(queueJson['post']['capability'], list):
  2003. if debug:
  2004. print('DEBUG: capability on post should be a list')
  2005. if os.path.isfile(queueFilename):
  2006. os.remove(queueFilename)
  2007. if len(queue)>0:
  2008. queue.pop(0)
  2009. continue
  2010. # Copy any posts addressed to followers into the shared inbox
  2011. # this avoid copying file multiple times to potentially many
  2012. # individual inboxes
  2013. # This obviously bypasses object capabilities and so
  2014. # any checking will needs to be handled at the time when inbox
  2015. # GET happens on individual accounts.
  2016. # See posts.py/createBoxBase
  2017. if len(recipientsDictFollowers)>0:
  2018. sharedInboxPostFilename=queueJson['destination'].replace(inboxHandle,inboxHandle)
  2019. if not os.path.isfile(sharedInboxPostFilename):
  2020. saveJson(queueJson['post'],sharedInboxPostFilename)
  2021. # for posts addressed to specific accounts
  2022. for handle,capsId in recipientsDict.items():
  2023. destination=queueJson['destination'].replace(inboxHandle,handle)
  2024. # check that capabilities are accepted
  2025. if queueJson['post'].get('capability'):
  2026. capabilityIdList=queueJson['post']['capability']
  2027. # does the capability id list within the post contain the id
  2028. # of the recipient with this handle?
  2029. # Here the capability id begins with the handle, so this could also
  2030. # be matched separately, but it's probably not necessary
  2031. if capsId in capabilityIdList:
  2032. inboxAfterCapabilities(session,keyId,handle, \
  2033. queueJson['post'], \
  2034. baseDir,httpPrefix, \
  2035. sendThreads,postLog, \
  2036. cachedWebfingers, \
  2037. personCache,queue,domain, \
  2038. port,useTor, \
  2039. federationList,ocapAlways, \
  2040. debug,acceptedCaps, \
  2041. queueFilename,destination, \
  2042. maxReplies,allowDeletion, \
  2043. maxMentions,maxEmoji, \
  2044. translate,unitTest)
  2045. else:
  2046. if debug:
  2047. print('DEBUG: object capabilities check has failed')
  2048. pprint(queueJson['post'])
  2049. else:
  2050. if not ocapAlways:
  2051. inboxAfterCapabilities(session,keyId,handle, \
  2052. queueJson['post'], \
  2053. baseDir,httpPrefix, \
  2054. sendThreads,postLog, \
  2055. cachedWebfingers, \
  2056. personCache,queue,domain, \
  2057. port,useTor, \
  2058. federationList,ocapAlways, \
  2059. debug,acceptedCaps, \
  2060. queueFilename,destination, \
  2061. maxReplies,allowDeletion, \
  2062. maxMentions,maxEmoji, \
  2063. translate,unitTest)
  2064. if debug:
  2065. pprint(queueJson['post'])
  2066. print('No capability list within post')
  2067. print('ocapAlways: '+str(ocapAlways))
  2068. print('DEBUG: object capabilities check failed')
  2069. if debug:
  2070. print('DEBUG: Queue post accepted')
  2071. if os.path.isfile(queueFilename):
  2072. os.remove(queueFilename)
  2073. if len(queue)>0:
  2074. queue.pop(0)