inbox.py 64 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957958959960961962963964965966967968969970971972973974975976977978979980981982983984985986987988989990991992993994995996997998999100010011002100310041005100610071008100910101011101210131014101510161017101810191020102110221023102410251026102710281029103010311032103310341035103610371038103910401041104210431044104510461047104810491050105110521053105410551056105710581059106010611062106310641065106610671068106910701071107210731074107510761077107810791080108110821083108410851086108710881089109010911092109310941095109610971098109911001101110211031104110511061107110811091110111111121113111411151116111711181119112011211122112311241125112611271128112911301131113211331134113511361137113811391140114111421143114411451146114711481149115011511152115311541155115611571158115911601161116211631164116511661167116811691170117111721173117411751176117711781179118011811182118311841185118611871188118911901191119211931194119511961197119811991200120112021203120412051206120712081209121012111212121312141215121612171218121912201221122212231224122512261227122812291230123112321233123412351236123712381239124012411242124312441245124612471248124912501251125212531254125512561257125812591260126112621263126412651266126712681269127012711272127312741275127612771278127912801281128212831284128512861287128812891290129112921293129412951296129712981299130013011302130313041305130613071308130913101311131213131314131513161317131813191320132113221323132413251326132713281329133013311332133313341335133613371338133913401341134213431344134513461347134813491350135113521353135413551356135713581359136013611362136313641365136613671368136913701371137213731374137513761377137813791380138113821383138413851386138713881389139013911392139313941395139613971398139914001401140214031404140514061407140814091410141114121413141414151416141714181419142014211422142314241425142614271428142914301431143214331434143514361437143814391440144114421443144414451446144714481449145014511452145314541455145614571458145914601461146214631464146514661467146814691470147114721473147414751476147714781479148014811482148314841485148614871488148914901491149214931494149514961497149814991500150115021503150415051506150715081509151015111512151315141515151615171518151915201521152215231524152515261527152815291530153115321533153415351536153715381539154015411542154315441545154615471548154915501551
  1. __filename__ = "inbox.py"
  2. __author__ = "Bob Mottram"
  3. __license__ = "AGPL3+"
  4. __version__ = "1.0.0"
  5. __maintainer__ = "Bob Mottram"
  6. __email__ = "bob@freedombone.net"
  7. __status__ = "Production"
  8. import json
  9. import os
  10. import datetime
  11. import time
  12. import json
  13. import commentjson
  14. from shutil import copyfile
  15. from utils import urlPermitted
  16. from utils import createInboxQueueDir
  17. from utils import getStatusNumber
  18. from utils import getDomainFromActor
  19. from utils import getNicknameFromActor
  20. from utils import domainPermitted
  21. from utils import locatePost
  22. from utils import deletePost
  23. from utils import removeAttachment
  24. from utils import removeModerationPostFromIndex
  25. from httpsig import verifyPostHeaders
  26. from session import createSession
  27. from session import getJson
  28. from follow import receiveFollowRequest
  29. from follow import getFollowersOfActor
  30. from follow import unfollowerOfPerson
  31. from pprint import pprint
  32. from cache import getPersonFromCache
  33. from cache import storePersonInCache
  34. from acceptreject import receiveAcceptReject
  35. from capabilities import getOcapFilename
  36. from capabilities import CapablePost
  37. from capabilities import capabilitiesReceiveUpdate
  38. from like import updateLikesCollection
  39. from like import undoLikesCollectionEntry
  40. from blocking import isBlocked
  41. from filters import isFiltered
  42. from announce import updateAnnounceCollection
  43. from httpsig import messageContentDigest
  44. from blocking import isBlockedDomain
  45. def validInbox(baseDir: str,nickname: str,domain: str) -> bool:
  46. """Checks whether files were correctly saved to the inbox
  47. """
  48. if ':' in domain:
  49. domain=domain.split(':')[0]
  50. inboxDir=baseDir+'/accounts/'+nickname+'@'+domain+'/inbox'
  51. if not os.path.isdir(inboxDir):
  52. return True
  53. for subdir, dirs, files in os.walk(inboxDir):
  54. for f in files:
  55. filename = os.path.join(subdir, f)
  56. if not os.path.isfile(filename):
  57. print('filename: '+filename)
  58. return False
  59. if 'postNickname' in open(filename).read():
  60. print('queue file incorrectly saved to '+filename)
  61. return False
  62. return True
  63. def validInboxFilenames(baseDir: str,nickname: str,domain: str, \
  64. expectedDomain: str,expectedPort: int) -> bool:
  65. """Used by unit tests to check that the port number gets appended to
  66. domain names within saved post filenames
  67. """
  68. if ':' in domain:
  69. domain=domain.split(':')[0]
  70. inboxDir=baseDir+'/accounts/'+nickname+'@'+domain+'/inbox'
  71. if not os.path.isdir(inboxDir):
  72. return True
  73. expectedStr=expectedDomain+':'+str(expectedPort)
  74. for subdir, dirs, files in os.walk(inboxDir):
  75. for f in files:
  76. filename = os.path.join(subdir, f)
  77. if not os.path.isfile(filename):
  78. print('filename: '+filename)
  79. return False
  80. if not expectedStr in filename:
  81. print('Expected: '+expectedStr)
  82. print('Invalid filename: '+filename)
  83. return False
  84. return True
  85. def getPersonPubKey(baseDir: str,session,personUrl: str, \
  86. personCache: {},debug: bool, \
  87. projectVersion: str,httpPrefix: str,domain: str) -> str:
  88. if not personUrl:
  89. return None
  90. personUrl=personUrl.replace('#main-key','')
  91. if personUrl.endswith('/users/inbox'):
  92. if debug:
  93. print('DEBUG: Obtaining public key for shared inbox')
  94. personUrl=personUrl.replace('/users/inbox','/inbox')
  95. personJson = getPersonFromCache(baseDir,personUrl,personCache)
  96. if not personJson:
  97. if debug:
  98. print('DEBUG: Obtaining public key for '+personUrl)
  99. asHeader = {'Accept': 'application/activity+json; profile="https://www.w3.org/ns/activitystreams"'}
  100. personJson = getJson(session,personUrl,asHeader,None,projectVersion,httpPrefix,domain)
  101. if not personJson:
  102. return None
  103. pubKey=None
  104. if personJson.get('publicKey'):
  105. if personJson['publicKey'].get('publicKeyPem'):
  106. pubKey=personJson['publicKey']['publicKeyPem']
  107. else:
  108. if personJson.get('publicKeyPem'):
  109. pubKey=personJson['publicKeyPem']
  110. if not pubKey:
  111. if debug:
  112. print('DEBUG: Public key not found for '+personUrl)
  113. storePersonInCache(baseDir,personUrl,personJson,personCache)
  114. return pubKey
  115. def inboxMessageHasParams(messageJson: {}) -> bool:
  116. """Checks whether an incoming message contains expected parameters
  117. """
  118. expectedParams=['type','actor','object']
  119. for param in expectedParams:
  120. if not messageJson.get(param):
  121. return False
  122. if not messageJson.get('to'):
  123. allowedWithoutToParam=['Like','Follow','Request','Accept','Capability','Undo']
  124. if messageJson['type'] not in allowedWithoutToParam:
  125. return False
  126. return True
  127. def inboxPermittedMessage(domain: str,messageJson: {},federationList: []) -> bool:
  128. """ check that we are receiving from a permitted domain
  129. """
  130. testParam='actor'
  131. if not messageJson.get(testParam):
  132. return False
  133. actor=messageJson[testParam]
  134. # always allow the local domain
  135. if domain in actor:
  136. return True
  137. if not urlPermitted(actor,federationList,"inbox:write"):
  138. return False
  139. if messageJson['type']!='Follow' and \
  140. messageJson['type']!='Like' and \
  141. messageJson['type']!='Delete' and \
  142. messageJson['type']!='Announce':
  143. if messageJson.get('object'):
  144. if not isinstance(messageJson['object'], dict):
  145. return False
  146. if messageJson['object'].get('inReplyTo'):
  147. inReplyTo=messageJson['object']['inReplyTo']
  148. if not urlPermitted(inReplyTo,federationList,"inbox:write"):
  149. return False
  150. return True
  151. def validPublishedDate(published) -> bool:
  152. currTime=datetime.datetime.utcnow()
  153. pubDate=datetime.datetime.strptime(published,"%Y-%m-%dT%H:%M:%SZ")
  154. daysSincePublished = (currTime - pubTime).days
  155. if daysSincePublished>30:
  156. return False
  157. return True
  158. def savePostToInboxQueue(baseDir: str,httpPrefix: str, \
  159. nickname: str, domain: str, \
  160. postJsonObject: {}, \
  161. messageBytes: str, \
  162. httpHeaders: {}, \
  163. postPath: str,debug: bool) -> str:
  164. """Saves the give json to the inbox queue for the person
  165. keyId specifies the actor sending the post
  166. """
  167. originalDomain=domain
  168. if ':' in domain:
  169. domain=domain.split(':')[0]
  170. # block at the ealiest stage possible, which means the data
  171. # isn't written to file
  172. postNickname=None
  173. postDomain=None
  174. actor=None
  175. if postJsonObject.get('actor'):
  176. actor=postJsonObject['actor']
  177. postNickname=getNicknameFromActor(postJsonObject['actor'])
  178. if not postNickname:
  179. print('No post Nickname in actor '+postJsonObject['actor'])
  180. return None
  181. postDomain,postPort=getDomainFromActor(postJsonObject['actor'])
  182. if not postDomain:
  183. pprint(postJsonObject)
  184. print('No post Domain in actor')
  185. return None
  186. if isBlocked(baseDir,nickname,domain,postNickname,postDomain):
  187. if debug:
  188. print('DEBUG: post from '+postNickname+' blocked')
  189. return None
  190. if postPort:
  191. if postPort!=80 and postPort!=443:
  192. if ':' not in postDomain:
  193. postDomain=postDomain+':'+str(postPort)
  194. if postJsonObject.get('object'):
  195. if isinstance(postJsonObject['object'], dict):
  196. if postJsonObject['object'].get('inReplyTo'):
  197. if isinstance(postJsonObject['object']['inReplyTo'], str):
  198. replyNickname=getNicknameFromActor(postJsonObject['object']['inReplyTo'])
  199. replyDomain,replyPort=getDomainFromActor(postJsonObject['object']['inReplyTo'])
  200. if replyNickname and replyDomain:
  201. if isBlocked(baseDir,nickname,domain,replyNickname,replyDomain):
  202. print('WARN: post contains reply to a blocked account: '+replyNickname+'@'+replyDomain)
  203. return None
  204. else:
  205. print('WARN: post is a reply to an unidentified account: '+postJsonObject['object']['inReplyTo'])
  206. return None
  207. if postJsonObject['object'].get('content'):
  208. if isinstance(postJsonObject['object']['content'], str):
  209. if isFiltered(baseDir,nickname,domain,postJsonObject['object']['content']):
  210. print('WARN: post was filtered out due to content')
  211. return None
  212. originalPostId=None
  213. if postJsonObject.get('id'):
  214. originalPostId=postJsonObject['id'].replace('/activity','').replace('/undo','')
  215. currTime=datetime.datetime.utcnow()
  216. postId=None
  217. if postJsonObject.get('id'):
  218. #if '/statuses/' not in postJsonObject['id']:
  219. postId=postJsonObject['id'].replace('/activity','').replace('/undo','')
  220. published=currTime.strftime("%Y-%m-%dT%H:%M:%SZ")
  221. if not postId:
  222. statusNumber,published = getStatusNumber()
  223. if actor:
  224. postId=actor+'/statuses/'+statusNumber
  225. else:
  226. postId=httpPrefix+'://'+originalDomain+'/users/'+nickname+'/statuses/'+statusNumber
  227. # NOTE: don't change postJsonObject['id'] before signature check
  228. inboxQueueDir=createInboxQueueDir(nickname,domain,baseDir)
  229. handle=nickname+'@'+domain
  230. destination=baseDir+'/accounts/'+handle+'/inbox/'+postId.replace('/','#')+'.json'
  231. #if os.path.isfile(destination):
  232. # if debug:
  233. # print(destination)
  234. # print('DEBUG: inbox item already exists')
  235. # return None
  236. filename=inboxQueueDir+'/'+postId.replace('/','#')+'.json'
  237. sharedInboxItem=False
  238. if nickname=='inbox':
  239. nickname=originalDomain
  240. sharedInboxItem=True
  241. newQueueItem = {
  242. 'originalId': originalPostId,
  243. 'id': postId,
  244. 'actor': actor,
  245. 'nickname': nickname,
  246. 'domain': domain,
  247. 'postNickname': postNickname,
  248. 'postDomain': postDomain,
  249. 'sharedInbox': sharedInboxItem,
  250. 'published': published,
  251. 'httpHeaders': httpHeaders,
  252. 'path': postPath,
  253. 'post': postJsonObject,
  254. 'digest': messageContentDigest(messageBytes),
  255. 'filename': filename,
  256. 'destination': destination
  257. }
  258. if debug:
  259. print('Inbox queue item created')
  260. pprint(newQueueItem)
  261. with open(filename, 'w') as fp:
  262. commentjson.dump(newQueueItem, fp, indent=4, sort_keys=False)
  263. return filename
  264. def inboxCheckCapabilities(baseDir :str,nickname :str,domain :str, \
  265. actor: str,queue: [],queueJson: {}, \
  266. capabilityId: str,debug : bool) -> bool:
  267. if nickname=='inbox':
  268. return True
  269. ocapFilename= \
  270. getOcapFilename(baseDir, \
  271. queueJson['nickname'],queueJson['domain'], \
  272. actor,'accept')
  273. if not ocapFilename:
  274. return False
  275. if not os.path.isfile(ocapFilename):
  276. if debug:
  277. print('DEBUG: capabilities for '+ \
  278. actor+' do not exist')
  279. os.remove(queueFilename)
  280. queue.pop(0)
  281. return False
  282. with open(ocapFilename, 'r') as fp:
  283. oc=commentjson.load(fp)
  284. if not oc.get('id'):
  285. if debug:
  286. print('DEBUG: capabilities for '+actor+' do not contain an id')
  287. os.remove(queueFilename)
  288. queue.pop(0)
  289. return False
  290. if oc['id']!=capabilityId:
  291. if debug:
  292. print('DEBUG: capability id mismatch')
  293. os.remove(queueFilename)
  294. queue.pop(0)
  295. return False
  296. if not oc.get('capability'):
  297. if debug:
  298. print('DEBUG: missing capability list')
  299. os.remove(queueFilename)
  300. queue.pop(0)
  301. return False
  302. if not CapablePost(queueJson['post'],oc['capability'],debug):
  303. if debug:
  304. print('DEBUG: insufficient capabilities to write to inbox from '+actor)
  305. os.remove(queueFilename)
  306. queue.pop(0)
  307. return False
  308. if debug:
  309. print('DEBUG: object capabilities check success')
  310. return True
  311. def inboxPostRecipientsAdd(baseDir :str,httpPrefix :str,toList :[], \
  312. recipientsDict :{}, \
  313. domainMatch: str,domain :str, \
  314. actor :str,debug: bool) -> bool:
  315. """Given a list of post recipients (toList) from 'to' or 'cc' parameters
  316. populate a recipientsDict with the handle and capabilities id for each
  317. """
  318. followerRecipients=False
  319. for recipient in toList:
  320. if not recipient:
  321. continue
  322. # is this a to a local account?
  323. if domainMatch in recipient:
  324. # get the handle for the local account
  325. nickname=recipient.split(domainMatch)[1]
  326. handle=nickname+'@'+domain
  327. if os.path.isdir(baseDir+'/accounts/'+handle):
  328. # are capabilities granted for this account to the
  329. # sender (actor) of the post?
  330. ocapFilename=baseDir+'/accounts/'+handle+'/ocap/accept/'+actor.replace('/','#')+'.json'
  331. if os.path.isfile(ocapFilename):
  332. # read the granted capabilities and obtain the id
  333. with open(ocapFilename, 'r') as fp:
  334. ocapJson=commentjson.load(fp)
  335. if ocapJson.get('id'):
  336. # append with the capabilities id
  337. recipientsDict[handle]=ocapJson['id']
  338. else:
  339. recipientsDict[handle]=None
  340. else:
  341. if debug:
  342. print('DEBUG: '+ocapFilename+' not found')
  343. recipientsDict[handle]=None
  344. else:
  345. if debug:
  346. print('DEBUG: '+baseDir+'/accounts/'+handle+' does not exist')
  347. else:
  348. if debug:
  349. print('DEBUG: '+recipient+' is not local to '+domainMatch)
  350. print(str(toList))
  351. if recipient.endswith('followers'):
  352. if debug:
  353. print('DEBUG: followers detected as post recipients')
  354. followerRecipients=True
  355. return followerRecipients,recipientsDict
  356. def inboxPostRecipients(baseDir :str,postJsonObject :{},httpPrefix :str,domain : str,port :int, debug :bool) -> ([],[]):
  357. """Returns dictionaries containing the recipients of the given post
  358. The shared dictionary contains followers
  359. """
  360. recipientsDict={}
  361. recipientsDictFollowers={}
  362. if not postJsonObject.get('actor'):
  363. if debug:
  364. pprint(postJsonObject)
  365. print('WARNING: inbox post has no actor')
  366. return recipientsDict,recipientsDictFollowers
  367. if ':' in domain:
  368. domain=domain.split(':')[0]
  369. domainBase=domain
  370. if port:
  371. if port!=80 and port!=443:
  372. if ':' not in domain:
  373. domain=domain+':'+str(port)
  374. domainMatch='/'+domain+'/users/'
  375. actor = postJsonObject['actor']
  376. # first get any specific people which the post is addressed to
  377. followerRecipients=False
  378. if postJsonObject.get('object'):
  379. if isinstance(postJsonObject['object'], dict):
  380. if postJsonObject['object'].get('to'):
  381. if isinstance(postJsonObject['object']['to'], list):
  382. recipientsList=postJsonObject['object']['to']
  383. else:
  384. recipientsList=[postJsonObject['object']['to']]
  385. if debug:
  386. print('DEBUG: resolving "to"')
  387. includesFollowers,recipientsDict= \
  388. inboxPostRecipientsAdd(baseDir,httpPrefix, \
  389. recipientsList, \
  390. recipientsDict, \
  391. domainMatch,domainBase, \
  392. actor,debug)
  393. if includesFollowers:
  394. followerRecipients=True
  395. else:
  396. if debug:
  397. print('DEBUG: inbox post has no "to"')
  398. if postJsonObject['object'].get('cc'):
  399. if isinstance(postJsonObject['object']['cc'], list):
  400. recipientsList=postJsonObject['object']['cc']
  401. else:
  402. recipientsList=[postJsonObject['object']['cc']]
  403. includesFollowers,recipientsDict= \
  404. inboxPostRecipientsAdd(baseDir,httpPrefix, \
  405. recipientsList, \
  406. recipientsDict, \
  407. domainMatch,domainBase, \
  408. actor,debug)
  409. if includesFollowers:
  410. followerRecipients=True
  411. else:
  412. if debug:
  413. print('DEBUG: inbox post has no cc')
  414. else:
  415. if debug:
  416. if isinstance(postJsonObject['object'], str):
  417. if '/statuses/' in postJsonObject['object']:
  418. print('DEBUG: inbox item is a link to a post')
  419. else:
  420. if '/users/' in postJsonObject['object']:
  421. print('DEBUG: inbox item is a link to an actor')
  422. if postJsonObject.get('to'):
  423. if isinstance(postJsonObject['to'], list):
  424. recipientsList=postJsonObject['to']
  425. else:
  426. recipientsList=[postJsonObject['to']]
  427. includesFollowers,recipientsDict= \
  428. inboxPostRecipientsAdd(baseDir,httpPrefix, \
  429. recipientsList, \
  430. recipientsDict, \
  431. domainMatch,domainBase, \
  432. actor,debug)
  433. if includesFollowers:
  434. followerRecipients=True
  435. if postJsonObject.get('cc'):
  436. if isinstance(postJsonObject['cc'], list):
  437. recipientsList=postJsonObject['cc']
  438. else:
  439. recipientsList=[postJsonObject['cc']]
  440. includesFollowers,recipientsDict= \
  441. inboxPostRecipientsAdd(baseDir,httpPrefix, \
  442. recipientsList, \
  443. recipientsDict, \
  444. domainMatch,domainBase, \
  445. actor,debug)
  446. if includesFollowers:
  447. followerRecipients=True
  448. if not followerRecipients:
  449. if debug:
  450. print('DEBUG: no followers were resolved')
  451. return recipientsDict,recipientsDictFollowers
  452. # now resolve the followers
  453. recipientsDictFollowers= \
  454. getFollowersOfActor(baseDir,actor,debug)
  455. return recipientsDict,recipientsDictFollowers
  456. def receiveUndoFollow(session,baseDir: str,httpPrefix: str, \
  457. port: int,messageJson: {}, \
  458. federationList: [], \
  459. debug : bool) -> bool:
  460. if not messageJson['object'].get('actor'):
  461. if debug:
  462. print('DEBUG: follow request has no actor within object')
  463. return False
  464. if '/users/' not in messageJson['object']['actor'] and '/profile/' not in messageJson['object']['actor']:
  465. if debug:
  466. print('DEBUG: "users" or "profile" missing from actor within object')
  467. return False
  468. if messageJson['object']['actor'] != messageJson['actor']:
  469. if debug:
  470. print('DEBUG: actors do not match')
  471. return False
  472. nicknameFollower=getNicknameFromActor(messageJson['object']['actor'])
  473. if not nicknameFollower:
  474. print('WARN: unable to find nickname in '+messageJson['object']['actor'])
  475. return False
  476. domainFollower,portFollower=getDomainFromActor(messageJson['object']['actor'])
  477. domainFollowerFull=domainFollower
  478. if portFollower:
  479. if portFollower!=80 and portFollower!=443:
  480. if ':' not in domainFollower:
  481. domainFollowerFull=domainFollower+':'+str(portFollower)
  482. nicknameFollowing=getNicknameFromActor(messageJson['object']['object'])
  483. if not nicknameFollowing:
  484. print('WARN: unable to find nickname in '+messageJson['object']['object'])
  485. return False
  486. domainFollowing,portFollowing=getDomainFromActor(messageJson['object']['object'])
  487. domainFollowingFull=domainFollowing
  488. if portFollowing:
  489. if portFollowing!=80 and portFollowing!=443:
  490. if ':' not in domainFollowing:
  491. domainFollowingFull=domainFollowing+':'+str(portFollowing)
  492. if unfollowerOfPerson(baseDir, \
  493. nicknameFollowing,domainFollowingFull, \
  494. nicknameFollower,domainFollowerFull, \
  495. debug):
  496. if debug:
  497. print('DEBUG: Follower '+nicknameFollower+'@'+domainFollowerFull+' was removed')
  498. return True
  499. if debug:
  500. print('DEBUG: Follower '+nicknameFollower+'@'+domainFollowerFull+' was not removed')
  501. return False
  502. def receiveUndo(session,baseDir: str,httpPrefix: str, \
  503. port: int,sendThreads: [],postLog: [], \
  504. cachedWebfingers: {},personCache: {}, \
  505. messageJson: {},federationList: [], \
  506. debug : bool, \
  507. acceptedCaps=["inbox:write","objects:read"]) -> bool:
  508. """Receives an undo request within the POST section of HTTPServer
  509. """
  510. if not messageJson['type'].startswith('Undo'):
  511. return False
  512. if debug:
  513. print('DEBUG: Undo activity received')
  514. if not messageJson.get('actor'):
  515. if debug:
  516. print('DEBUG: follow request has no actor')
  517. return False
  518. if '/users/' not in messageJson['actor'] and '/profile/' not in messageJson['actor']:
  519. if debug:
  520. print('DEBUG: "users" or "profile" missing from actor')
  521. return False
  522. if not messageJson.get('object'):
  523. if debug:
  524. print('DEBUG: '+messageJson['type']+' has no object')
  525. return False
  526. if not isinstance(messageJson['object'], dict):
  527. if debug:
  528. print('DEBUG: '+messageJson['type']+' object is not a dict')
  529. return False
  530. if not messageJson['object'].get('type'):
  531. if debug:
  532. print('DEBUG: '+messageJson['type']+' has no object type')
  533. return False
  534. if not messageJson['object'].get('object'):
  535. if debug:
  536. print('DEBUG: '+messageJson['type']+' has no object within object')
  537. return False
  538. if not isinstance(messageJson['object']['object'], str):
  539. if debug:
  540. print('DEBUG: '+messageJson['type']+' object within object is not a string')
  541. return False
  542. if messageJson['object']['type']=='Follow':
  543. return receiveUndoFollow(session,baseDir,httpPrefix, \
  544. port,messageJson, \
  545. federationList, \
  546. debug)
  547. return False
  548. def personReceiveUpdate(baseDir: str, \
  549. domain: str,port: int, \
  550. updateNickname: str,updateDomain: str,updatePort: int, \
  551. personJson: {},personCache: {},debug: bool) -> bool:
  552. """Changes an actor. eg: avatar or display name change
  553. """
  554. if debug:
  555. print('DEBUG: receiving actor update for '+personJson['url'])
  556. domainFull=domain
  557. if port:
  558. if port!=80 and port!=443:
  559. domainFull=domain+':'+str(port)
  560. updateDomainFull=updateDomain
  561. if updatePort:
  562. if updatePort!=80 and updatePort!=443:
  563. updateDomainFull=updateDomain+':'+str(updatePort)
  564. actor=updateDomainFull+'/users/'+updateNickname
  565. if actor not in personJson['id']:
  566. actor=updateDomainFull+'/profile/'+updateNickname
  567. if actor not in personJson['id']:
  568. if debug:
  569. print('actor: '+actor)
  570. print('id: '+personJson['id'])
  571. print('DEBUG: Actor does not match id')
  572. return False
  573. if updateDomainFull==domainFull:
  574. if debug:
  575. print('DEBUG: You can only receive actor updates for domains other than your own')
  576. return False
  577. if not personJson.get('publicKey'):
  578. if debug:
  579. print('DEBUG: actor update does not contain a public key')
  580. return False
  581. if not personJson['publicKey'].get('publicKeyPem'):
  582. if debug:
  583. print('DEBUG: actor update does not contain a public key Pem')
  584. return False
  585. actorFilename=baseDir+'/cache/actors/'+personJson['id'].replace('/','#')+'.json'
  586. # check that the public keys match.
  587. # If they don't then this may be a nefarious attempt to hack an account
  588. if personCache.get(personJson['id']):
  589. if personCache[personJson['id']]['actor']['publicKey']['publicKeyPem']!=personJson['publicKey']['publicKeyPem']:
  590. if debug:
  591. print('WARN: Public key does not match when updating actor')
  592. return False
  593. else:
  594. if os.path.isfile(actorFilename):
  595. with open(actorFilename, 'r') as fp:
  596. existingPersonJson=commentjson.load(fp)
  597. if existingPersonJson['publicKey']['publicKeyPem']!=personJson['publicKey']['publicKeyPem']:
  598. if debug:
  599. print('WARN: Public key does not match cached actor when updating')
  600. return False
  601. # save to cache in memory
  602. storePersonInCache(baseDir,personJson['id'],personJson,personCache)
  603. # save to cache on file
  604. with open(actorFilename, 'w') as fp:
  605. commentjson.dump(personJson, fp, indent=4, sort_keys=False)
  606. print('actor updated for '+personJson['id'])
  607. # remove avatar if it exists so that it will be refreshed later
  608. # when a timeline is constructed
  609. actorStr=personJson['id'].replace('/','-')
  610. avatarFilename=baseDir+'/cache/avatars/'+actorStr+'.png'
  611. if os.path.isfile(avatarFilename):
  612. os.remove(avatarFilename)
  613. else:
  614. avatarFilename=baseDir+'/cache/avatars/'+actorStr+'.jpg'
  615. if os.path.isfile(avatarFilename):
  616. os.remove(avatarFilename)
  617. else:
  618. avatarFilename=baseDir+'/cache/avatars/'+actorStr+'.gif'
  619. if os.path.isfile(avatarFilename):
  620. os.remove(avatarFilename)
  621. return True
  622. def receiveUpdate(session,baseDir: str, \
  623. httpPrefix: str,domain :str,port: int, \
  624. sendThreads: [],postLog: [],cachedWebfingers: {}, \
  625. personCache: {},messageJson: {},federationList: [], \
  626. debug : bool) -> bool:
  627. """Receives an Update activity within the POST section of HTTPServer
  628. """
  629. if messageJson['type']!='Update':
  630. return False
  631. if not messageJson.get('actor'):
  632. if debug:
  633. print('DEBUG: '+messageJson['type']+' has no actor')
  634. return False
  635. if not messageJson.get('object'):
  636. if debug:
  637. print('DEBUG: '+messageJson['type']+' has no object')
  638. return False
  639. if not isinstance(messageJson['object'], dict):
  640. if debug:
  641. print('DEBUG: '+messageJson['type']+' object is not a dict')
  642. return False
  643. if not messageJson['object'].get('type'):
  644. if debug:
  645. print('DEBUG: '+messageJson['type']+' object has no type')
  646. return False
  647. if '/users/' not in messageJson['actor'] and '/profile/' not in messageJson['actor']:
  648. if debug:
  649. print('DEBUG: "users" or "profile" missing from actor in '+messageJson['type'])
  650. return False
  651. if messageJson['object']['type']=='Person' or \
  652. messageJson['object']['type']=='Application' or \
  653. messageJson['object']['type']=='Service':
  654. if messageJson['object'].get('url') and messageJson['object'].get('id'):
  655. print('Request to update actor: '+messageJson['actor'])
  656. updateNickname=getNicknameFromActor(messageJson['actor'])
  657. if updateNickname:
  658. updateDomain,updatePort=getDomainFromActor(messageJson['actor'])
  659. if personReceiveUpdate(baseDir, \
  660. domain,port, \
  661. updateNickname,updateDomain,updatePort, \
  662. messageJson['object'], \
  663. personCache,debug):
  664. if debug:
  665. print('DEBUG: Profile update was received for '+messageJson['object']['url'])
  666. return True
  667. if messageJson['object'].get('capability') and messageJson['object'].get('scope'):
  668. nickname=getNicknameFromActor(messageJson['object']['scope'])
  669. if nickname:
  670. domain,tempPort=getDomainFromActor(messageJson['object']['scope'])
  671. if messageJson['object']['type']=='Capability':
  672. if capabilitiesReceiveUpdate(baseDir,nickname,domain,port,
  673. messageJson['actor'], \
  674. messageJson['object']['id'], \
  675. messageJson['object']['capability'], \
  676. debug):
  677. if debug:
  678. print('DEBUG: An update was received')
  679. return True
  680. return False
  681. def receiveLike(session,handle: str,baseDir: str, \
  682. httpPrefix: str,domain :str,port: int, \
  683. sendThreads: [],postLog: [],cachedWebfingers: {}, \
  684. personCache: {},messageJson: {},federationList: [], \
  685. debug : bool) -> bool:
  686. """Receives a Like activity within the POST section of HTTPServer
  687. """
  688. if messageJson['type']!='Like':
  689. return False
  690. if not messageJson.get('actor'):
  691. if debug:
  692. print('DEBUG: '+messageJson['type']+' has no actor')
  693. return False
  694. if not messageJson.get('object'):
  695. if debug:
  696. print('DEBUG: '+messageJson['type']+' has no object')
  697. return False
  698. if not isinstance(messageJson['object'], str):
  699. if debug:
  700. print('DEBUG: '+messageJson['type']+' object is not a string')
  701. return False
  702. if not messageJson.get('to'):
  703. if debug:
  704. print('DEBUG: '+messageJson['type']+' has no "to" list')
  705. return False
  706. if '/users/' not in messageJson['actor'] and '/profile/' not in messageJson['actor']:
  707. if debug:
  708. print('DEBUG: "users" or "profile" missing from actor in '+messageJson['type'])
  709. return False
  710. if '/statuses/' not in messageJson['object']:
  711. if debug:
  712. print('DEBUG: "statuses" missing from object in '+messageJson['type'])
  713. return False
  714. if not os.path.isdir(baseDir+'/accounts/'+handle):
  715. print('DEBUG: unknown recipient of like - '+handle)
  716. # if this post in the outbox of the person?
  717. postFilename=locatePost(baseDir,handle.split('@')[0],handle.split('@')[1],messageJson['object'])
  718. if not postFilename:
  719. if debug:
  720. print('DEBUG: post not found in inbox or outbox')
  721. print(messageJson['object'])
  722. return True
  723. if debug:
  724. print('DEBUG: liked post found in inbox')
  725. updateLikesCollection(postFilename,messageJson['object'],messageJson['actor'],debug)
  726. return True
  727. def receiveUndoLike(session,handle: str,baseDir: str, \
  728. httpPrefix: str,domain :str,port: int, \
  729. sendThreads: [],postLog: [],cachedWebfingers: {}, \
  730. personCache: {},messageJson: {},federationList: [], \
  731. debug : bool) -> bool:
  732. """Receives an undo like activity within the POST section of HTTPServer
  733. """
  734. if messageJson['type']!='Undo':
  735. return False
  736. if not messageJson.get('actor'):
  737. return False
  738. if not messageJson.get('object'):
  739. return False
  740. if not isinstance(messageJson['object'], dict):
  741. return False
  742. if not messageJson['object'].get('type'):
  743. return False
  744. if messageJson['object']['type']!='Like':
  745. return False
  746. if not messageJson['object'].get('object'):
  747. if debug:
  748. print('DEBUG: '+messageJson['type']+' like has no object')
  749. return False
  750. if not isinstance(messageJson['object']['object'], str):
  751. if debug:
  752. print('DEBUG: '+messageJson['type']+' like object is not a string')
  753. return False
  754. if '/users/' not in messageJson['actor'] and '/profile/' not in messageJson['actor']:
  755. if debug:
  756. print('DEBUG: "users" or "profile" missing from actor in '+messageJson['type']+' like')
  757. return False
  758. if '/statuses/' not in messageJson['object']['object']:
  759. if debug:
  760. print('DEBUG: "statuses" missing from like object in '+messageJson['type'])
  761. return False
  762. if not os.path.isdir(baseDir+'/accounts/'+handle):
  763. print('DEBUG: unknown recipient of undo like - '+handle)
  764. # if this post in the outbox of the person?
  765. postFilename=locatePost(baseDir,handle.split('@')[0],handle.split('@')[1],messageJson['object']['object'])
  766. if not postFilename:
  767. if debug:
  768. print('DEBUG: unliked post not found in inbox or outbox')
  769. print(messageJson['object']['object'])
  770. return True
  771. if debug:
  772. print('DEBUG: liked post found in inbox. Now undoing.')
  773. undoLikesCollectionEntry(postFilename,messageJson['object'],messageJson['actor'],debug)
  774. return True
  775. def receiveDelete(session,handle: str,baseDir: str, \
  776. httpPrefix: str,domain :str,port: int, \
  777. sendThreads: [],postLog: [],cachedWebfingers: {}, \
  778. personCache: {},messageJson: {},federationList: [], \
  779. debug : bool,allowDeletion: bool) -> bool:
  780. """Receives a Delete activity within the POST section of HTTPServer
  781. """
  782. if messageJson['type']!='Delete':
  783. return False
  784. if not messageJson.get('actor'):
  785. if debug:
  786. print('DEBUG: '+messageJson['type']+' has no actor')
  787. return False
  788. if debug:
  789. print('DEBUG: Delete activity arrived')
  790. if not messageJson.get('object'):
  791. if debug:
  792. print('DEBUG: '+messageJson['type']+' has no object')
  793. return False
  794. if not isinstance(messageJson['object'], str):
  795. if debug:
  796. print('DEBUG: '+messageJson['type']+' object is not a string')
  797. return False
  798. domainFull=domain
  799. if port:
  800. if port!=80 and port!=443:
  801. if ':' not in domain:
  802. domainFull=domain+':'+str(port)
  803. deletePrefix=httpPrefix+'://'+domainFull+'/'
  804. if not allowDeletion and \
  805. (not messageJson['object'].startswith(deletePrefix) or \
  806. not messageJson['actor'].startswith(deletePrefix)):
  807. if debug:
  808. print('DEBUG: delete not permitted from other instances')
  809. return False
  810. if not messageJson.get('to'):
  811. if debug:
  812. print('DEBUG: '+messageJson['type']+' has no "to" list')
  813. return False
  814. if '/users/' not in messageJson['actor'] and '/profile/' not in messageJson['actor']:
  815. if debug:
  816. print('DEBUG: "users" or "profile" missing from actor in '+messageJson['type'])
  817. return False
  818. if '/statuses/' not in messageJson['object']:
  819. if debug:
  820. print('DEBUG: "statuses" missing from object in '+messageJson['type'])
  821. return False
  822. if messageJson['actor'] not in messageJson['object']:
  823. if debug:
  824. print('DEBUG: actor is not the owner of the post to be deleted')
  825. if not os.path.isdir(baseDir+'/accounts/'+handle):
  826. print('DEBUG: unknown recipient of like - '+handle)
  827. # if this post in the outbox of the person?
  828. messageId=messageJson['object'].replace('/activity','').replace('/undo','')
  829. removeModerationPostFromIndex(baseDir,messageId,debug)
  830. postFilename=locatePost(baseDir,handle.split('@')[0],handle.split('@')[1],messageId)
  831. if not postFilename:
  832. if debug:
  833. print('DEBUG: delete post not found in inbox or outbox')
  834. print(messageId)
  835. return True
  836. deletePost(baseDir,httpPrefix,handle.split('@')[0],handle.split('@')[1],postFilename,debug)
  837. if debug:
  838. print('DEBUG: post deleted - '+postFilename)
  839. return True
  840. def receiveAnnounce(session,handle: str,baseDir: str, \
  841. httpPrefix: str,domain :str,port: int, \
  842. sendThreads: [],postLog: [],cachedWebfingers: {}, \
  843. personCache: {},messageJson: {},federationList: [], \
  844. debug : bool) -> bool:
  845. """Receives an announce activity within the POST section of HTTPServer
  846. """
  847. if messageJson['type']!='Announce':
  848. return False
  849. if not messageJson.get('actor'):
  850. if debug:
  851. print('DEBUG: '+messageJson['type']+' has no actor')
  852. return False
  853. if debug:
  854. print('DEBUG: receiving announce on '+handle)
  855. if not messageJson.get('object'):
  856. if debug:
  857. print('DEBUG: '+messageJson['type']+' has no object')
  858. return False
  859. if not isinstance(messageJson['object'], str):
  860. if debug:
  861. print('DEBUG: '+messageJson['type']+' object is not a string')
  862. return False
  863. if not messageJson.get('to'):
  864. if debug:
  865. print('DEBUG: '+messageJson['type']+' has no "to" list')
  866. return False
  867. if '/users/' not in messageJson['actor'] and '/profile/' not in messageJson['actor']:
  868. if debug:
  869. print('DEBUG: "users" or "profile" missing from actor in '+messageJson['type'])
  870. return False
  871. if '/users/' not in messageJson['object'] and '/profile/' not in messageJson['object']:
  872. if debug:
  873. print('DEBUG: "users" or "profile" missing in '+messageJson['type'])
  874. return False
  875. if '/statuses/' not in messageJson['object']:
  876. if debug:
  877. print('DEBUG: "statuses" missing from object in '+messageJson['type'])
  878. return False
  879. objectDomain=messageJson['object'].replace('https://','').replace('http://','').replace('dat://','')
  880. if '/' in objectDomain:
  881. objectDomain=objectDomain.split('/')[0]
  882. if isBlockedDomain(baseDir,objectDomain):
  883. if debug:
  884. print('DEBUG: announced domain is blocked')
  885. return False
  886. if not os.path.isdir(baseDir+'/accounts/'+handle):
  887. print('DEBUG: unknown recipient of announce - '+handle)
  888. # is this post in the outbox of the person?
  889. postFilename=locatePost(baseDir,handle.split('@')[0],handle.split('@')[1],messageJson['object'])
  890. if not postFilename:
  891. if debug:
  892. print('DEBUG: announce post not found in inbox or outbox')
  893. print(messageJson['object'])
  894. return True
  895. updateAnnounceCollection(postFilename,messageJson['actor'],debug)
  896. if debug:
  897. print('DEBUG: announced/repeated post found in inbox')
  898. return True
  899. def receiveUndoAnnounce(session,handle: str,baseDir: str, \
  900. httpPrefix: str,domain :str,port: int, \
  901. sendThreads: [],postLog: [],cachedWebfingers: {}, \
  902. personCache: {},messageJson: {},federationList: [], \
  903. debug : bool) -> bool:
  904. """Receives an undo announce activity within the POST section of HTTPServer
  905. """
  906. if messageJson['type']!='Undo':
  907. return False
  908. if not messageJson.get('actor'):
  909. return False
  910. if not messageJson.get('object'):
  911. return False
  912. if not isinstance(messageJson['object'], dict):
  913. return False
  914. if not messageJson['object'].get('object'):
  915. return False
  916. if not isinstance(messageJson['object']['object'], str):
  917. return False
  918. if messageJson['object']['type']!='Announce':
  919. return False
  920. if '/users/' not in messageJson['actor'] and '/profile/' not in messageJson['actor']:
  921. if debug:
  922. print('DEBUG: "users" or "profile" missing from actor in '+messageJson['type']+' announce')
  923. return False
  924. if '/statuses/' not in messageJson['object']:
  925. if debug:
  926. print('DEBUG: "statuses" missing from object in '+messageJson['type']+' announce')
  927. return False
  928. if not os.path.isdir(baseDir+'/accounts/'+handle):
  929. print('DEBUG: unknown recipient of undo announce - '+handle)
  930. # if this post in the outbox of the person?
  931. postFilename=locatePost(baseDir,handle.split('@')[0],handle.split('@')[1],messageJson['object'])
  932. if not postFilename:
  933. if debug:
  934. print('DEBUG: undo announce post not found in inbox or outbox')
  935. print(messageJson['object']['object'])
  936. return True
  937. if debug:
  938. print('DEBUG: announced/repeated post to be undone found in inbox')
  939. with open(postFilename, 'r') as fp:
  940. postJsonObject=commentjson.load(fp)
  941. if not postJsonObject.get('type'):
  942. if postJsonObject['type']!='Announce':
  943. if debug:
  944. print("DEBUG: Attempt to undo something which isn't an announcement")
  945. return False
  946. undoAnnounceCollectionEntry(postFilename,messageJson['actor'],debug)
  947. os.remove(postFilename)
  948. return True
  949. def populateReplies(baseDir :str,httpPrefix :str,domain :str, \
  950. messageJson :{},maxReplies: int,debug :bool) -> bool:
  951. """Updates the list of replies for a post on this domain if
  952. a reply to it arrives
  953. """
  954. if not messageJson.get('id'):
  955. return False
  956. if not messageJson.get('object'):
  957. return False
  958. if not isinstance(messageJson['object'], dict):
  959. return False
  960. if not messageJson['object'].get('inReplyTo'):
  961. return False
  962. if not messageJson['object'].get('to'):
  963. return False
  964. replyTo=messageJson['object']['inReplyTo']
  965. if debug:
  966. print('DEBUG: post contains a reply')
  967. # is this a reply to a post on this domain?
  968. if not replyTo.startswith(httpPrefix+'://'+domain+'/'):
  969. if debug:
  970. print('DEBUG: post is a reply to another not on this domain')
  971. print(replyTo)
  972. print('Expected: '+httpPrefix+'://'+domain+'/')
  973. return False
  974. replyToNickname=getNicknameFromActor(replyTo)
  975. if not replyToNickname:
  976. print('DEBUG: no nickname found for '+replyTo)
  977. return False
  978. replyToDomain,replyToPort=getDomainFromActor(replyTo)
  979. if not replyToDomain:
  980. if debug:
  981. print('DEBUG: no domain found for '+replyTo)
  982. return False
  983. postFilename=locatePost(baseDir,replyToNickname,replyToDomain,replyTo)
  984. if not postFilename:
  985. if debug:
  986. print('DEBUG: post may have expired - '+replyTo)
  987. return False
  988. # populate a text file containing the ids of replies
  989. postRepliesFilename=postFilename.replace('.json','.replies')
  990. messageId=messageJson['id'].replace('/activity','').replace('/undo','')
  991. if os.path.isfile(postRepliesFilename):
  992. numLines = sum(1 for line in open(postRepliesFilename))
  993. if numLines>maxReplies:
  994. return False
  995. if messageId not in open(postRepliesFilename).read():
  996. repliesFile=open(postRepliesFilename, "a")
  997. repliesFile.write(messageId+'\n')
  998. repliesFile.close()
  999. else:
  1000. repliesFile=open(postRepliesFilename, "w")
  1001. repliesFile.write(messageId+'\n')
  1002. repliesFile.close()
  1003. return True
  1004. def inboxAfterCapabilities(session,keyId: str,handle: str,messageJson: {}, \
  1005. baseDir: str,httpPrefix: str,sendThreads: [], \
  1006. postLog: [],cachedWebfingers: {},personCache: {}, \
  1007. queue: [],domain: str,port: int,useTor: bool, \
  1008. federationList: [],ocapAlways: bool,debug: bool, \
  1009. acceptedCaps: [],
  1010. queueFilename :str,destinationFilename :str,
  1011. maxReplies: int,allowDeletion: bool) -> bool:
  1012. """ Anything which needs to be done after capabilities checks have passed
  1013. """
  1014. if receiveLike(session,handle, \
  1015. baseDir,httpPrefix, \
  1016. domain,port, \
  1017. sendThreads,postLog, \
  1018. cachedWebfingers, \
  1019. personCache, \
  1020. messageJson, \
  1021. federationList, \
  1022. debug):
  1023. if debug:
  1024. print('DEBUG: Like accepted from '+keyId)
  1025. return False
  1026. if receiveUndoLike(session,handle, \
  1027. baseDir,httpPrefix, \
  1028. domain,port, \
  1029. sendThreads,postLog, \
  1030. cachedWebfingers, \
  1031. personCache, \
  1032. messageJson, \
  1033. federationList, \
  1034. debug):
  1035. if debug:
  1036. print('DEBUG: Undo like accepted from '+keyId)
  1037. return False
  1038. if receiveAnnounce(session,handle, \
  1039. baseDir,httpPrefix, \
  1040. domain,port, \
  1041. sendThreads,postLog, \
  1042. cachedWebfingers, \
  1043. personCache, \
  1044. messageJson, \
  1045. federationList, \
  1046. debug):
  1047. if debug:
  1048. print('DEBUG: Announce accepted from '+keyId)
  1049. if receiveUndoAnnounce(session,handle, \
  1050. baseDir,httpPrefix, \
  1051. domain,port, \
  1052. sendThreads,postLog, \
  1053. cachedWebfingers, \
  1054. personCache, \
  1055. messageJson, \
  1056. federationList, \
  1057. debug):
  1058. if debug:
  1059. print('DEBUG: Undo announce accepted from '+keyId)
  1060. return False
  1061. if receiveDelete(session,handle, \
  1062. baseDir,httpPrefix, \
  1063. domain,port, \
  1064. sendThreads,postLog, \
  1065. cachedWebfingers, \
  1066. personCache, \
  1067. messageJson, \
  1068. federationList, \
  1069. debug,allowDeletion):
  1070. if debug:
  1071. print('DEBUG: Delete accepted from '+keyId)
  1072. return False
  1073. populateReplies(baseDir,httpPrefix,domain,messageJson,maxReplies,debug)
  1074. if debug:
  1075. print('DEBUG: object capabilities passed')
  1076. print('copy queue file from '+queueFilename+' to '+destinationFilename)
  1077. if os.path.isfile(destinationFilename):
  1078. return True
  1079. if messageJson.get('postNickname'):
  1080. with open(destinationFilename, 'w+') as fp:
  1081. commentjson.dump(messageJson['post'], fp, indent=4, sort_keys=False)
  1082. else:
  1083. with open(destinationFilename, 'w+') as fp:
  1084. commentjson.dump(messageJson, fp, indent=4, sort_keys=False)
  1085. if not os.path.isfile(destinationFilename):
  1086. return False
  1087. return True
  1088. def restoreQueueItems(baseDir: str,queue: []) -> None:
  1089. """Checks the queue for each account and appends filenames
  1090. """
  1091. queue.clear()
  1092. for subdir,dirs,files in os.walk(baseDir+'/accounts'):
  1093. for account in dirs:
  1094. queueDir=baseDir+'/accounts/'+account+'/queue'
  1095. if os.path.isdir(queueDir):
  1096. for queuesubdir,queuedirs,queuefiles in os.walk(queueDir):
  1097. for qfile in queuefiles:
  1098. queue.append(os.path.join(queueDir, qfile))
  1099. if len(queue)>0:
  1100. print('Restored '+str(len(queue))+' inbox queue items')
  1101. def runInboxQueueWatchdog(projectVersion: str,httpd) -> None:
  1102. """This tries to keep the inbox thread running even if it dies
  1103. """
  1104. print('Starting inbox queue watchdog')
  1105. inboxQueueOriginal=httpd.thrInboxQueue.clone(runInboxQueue)
  1106. #httpd.thrInboxQueue=inboxQueueOriginal
  1107. httpd.thrInboxQueue.start()
  1108. while True:
  1109. time.sleep(20)
  1110. if not httpd.thrInboxQueue.isAlive():
  1111. httpd.thrInboxQueue.kill()
  1112. httpd.thrInboxQueue=inboxQueueOriginal.clone(runInboxQueue)
  1113. httpd.thrInboxQueue.start()
  1114. print('Restarting inbox queue...')
  1115. def runInboxQueue(projectVersion: str, \
  1116. baseDir: str,httpPrefix: str,sendThreads: [],postLog: [], \
  1117. cachedWebfingers: {},personCache: {},queue: [], \
  1118. domain: str,port: int,useTor: bool,federationList: [], \
  1119. ocapAlways: bool,maxReplies: int, \
  1120. domainMaxPostsPerDay: int,accountMaxPostsPerDay: int, \
  1121. allowDeletion: bool,debug: bool, \
  1122. acceptedCaps=["inbox:write","objects:read"]) -> None:
  1123. """Processes received items and moves them to
  1124. the appropriate directories
  1125. """
  1126. currSessionTime=int(time.time())
  1127. sessionLastUpdate=currSessionTime
  1128. session=createSession(domain,port,useTor)
  1129. inboxHandle='inbox@'+domain
  1130. if debug:
  1131. print('DEBUG: Inbox queue running')
  1132. # if queue processing was interrupted (eg server crash)
  1133. # then this loads any outstanding items back into the queue
  1134. restoreQueueItems(baseDir,queue)
  1135. # keep track of numbers of incoming posts per unit of time
  1136. quotasLastUpdate=int(time.time())
  1137. quotas={
  1138. 'domains': {},
  1139. 'accounts': {}
  1140. }
  1141. # keep track of the number of queue item read failures
  1142. # so that if a file is corrupt then it will eventually
  1143. # be ignored rather than endlessly retried
  1144. itemReadFailed=0
  1145. heartBeatCtr=0
  1146. queueRestoreCtr=0
  1147. while True:
  1148. time.sleep(1)
  1149. # heartbeat to monitor whether the inbox queue is running
  1150. heartBeatCtr+=1
  1151. if heartBeatCtr>=10:
  1152. print('>>> Heartbeat Q:'+str(len(queue))+' '+datetime.datetime.now().strftime("%m/%d/%Y, %H:%M:%S"))
  1153. heartBeatCtr=0
  1154. if len(queue)==0:
  1155. # restore any remaining queue items
  1156. queueRestoreCtr+=1
  1157. if queueRestoreCtr>=30:
  1158. queueRestoreCtr=0
  1159. restoreQueueItems(baseDir,queue)
  1160. else:
  1161. currTime=int(time.time())
  1162. # recreate the session periodically
  1163. if not session or currTime-sessionLastUpdate>1200:
  1164. print('Creating inbox session')
  1165. session=createSession(domain,port,useTor)
  1166. sessionLastUpdate=currTime
  1167. # oldest item first
  1168. queue.sort()
  1169. queueFilename=queue[0]
  1170. if not os.path.isfile(queueFilename):
  1171. if debug:
  1172. print("DEBUG: queue item rejected because it has no file: "+queueFilename)
  1173. queue.pop(0)
  1174. continue
  1175. print('Loading queue item '+queueFilename)
  1176. # Load the queue json
  1177. try:
  1178. with open(queueFilename, 'r') as fp:
  1179. queueJson=commentjson.load(fp)
  1180. except:
  1181. itemReadFailed+=1
  1182. print('WARN: Failed to load inbox queue item '+queueFilename+' (try '+str(itemReadFailed)+')')
  1183. if itemReadFailed>4:
  1184. # After a few tries we can assume that the file
  1185. # is probably corrupt/unreadable
  1186. queue.pop(0)
  1187. itemReadFailed=0
  1188. # delete the queue file
  1189. os.remove(queueFilename)
  1190. continue
  1191. itemReadFailed=0
  1192. # clear the daily quotas for maximum numbers of received posts
  1193. if currTime-quotasLastUpdate>60*60*24:
  1194. quotas={
  1195. 'domains': {},
  1196. 'accounts': {}
  1197. }
  1198. quotasLastUpdate=currTime
  1199. # limit the number of posts which can arrive per domain per day
  1200. postDomain=queueJson['postDomain']
  1201. if postDomain:
  1202. if domainMaxPostsPerDay>0:
  1203. if quotas['domains'].get(postDomain):
  1204. if quotas['domains'][postDomain]>domainMaxPostsPerDay:
  1205. if debug:
  1206. print('DEBUG: Maximum posts for '+postDomain+' reached')
  1207. queue.pop(0)
  1208. continue
  1209. quotas['domains'][postDomain]+=1
  1210. else:
  1211. quotas['domains'][postDomain]=1
  1212. if accountMaxPostsPerDay>0:
  1213. postHandle=queueJson['postNickname']+'@'+postDomain
  1214. if quotas['accounts'].get(postHandle):
  1215. if quotas['accounts'][postHandle]>accountMaxPostsPerDay:
  1216. if debug:
  1217. print('DEBUG: Maximum posts for '+postHandle+' reached')
  1218. queue.pop(0)
  1219. continue
  1220. quotas['accounts'][postHandle]+=1
  1221. else:
  1222. quotas['accounts'][postHandle]=1
  1223. if debug:
  1224. if accountMaxPostsPerDay>0 or domainMaxPostsPerDay>0:
  1225. pprint(quotas)
  1226. print('Obtaining public key for actor '+queueJson['actor'])
  1227. # Try a few times to obtain the public key
  1228. pubKey=None
  1229. keyId=None
  1230. for tries in range(8):
  1231. keyId=None
  1232. signatureParams=queueJson['httpHeaders']['signature'].split(',')
  1233. for signatureItem in signatureParams:
  1234. if signatureItem.startswith('keyId='):
  1235. if '"' in signatureItem:
  1236. keyId=signatureItem.split('"')[1]
  1237. break
  1238. if not keyId:
  1239. if debug:
  1240. print('DEBUG: No keyId in signature: '+ \
  1241. queueJson['httpHeaders']['signature'])
  1242. os.remove(queueFilename)
  1243. queue.pop(0)
  1244. continue
  1245. pubKey= \
  1246. getPersonPubKey(baseDir,session,keyId, \
  1247. personCache,debug, \
  1248. projectVersion,httpPrefix,domain)
  1249. if pubKey:
  1250. print('DEBUG: public key: '+str(pubKey))
  1251. break
  1252. if debug:
  1253. print('DEBUG: Retry '+str(tries+1)+ \
  1254. ' obtaining public key for '+keyId)
  1255. time.sleep(5)
  1256. if not pubKey:
  1257. if debug:
  1258. print('DEBUG: public key could not be obtained from '+keyId)
  1259. os.remove(queueFilename)
  1260. queue.pop(0)
  1261. continue
  1262. # check the signature
  1263. if debug:
  1264. print('DEBUG: checking http headers')
  1265. pprint(queueJson['httpHeaders'])
  1266. if not verifyPostHeaders(httpPrefix, \
  1267. pubKey, \
  1268. queueJson['httpHeaders'], \
  1269. queueJson['path'],False, \
  1270. queueJson['digest'], \
  1271. json.dumps(queueJson['post'])):
  1272. if debug:
  1273. print('DEBUG: Header signature check failed')
  1274. os.remove(queueFilename)
  1275. queue.pop(0)
  1276. continue
  1277. if debug:
  1278. print('DEBUG: Signature check success')
  1279. # set the id to the same as the post filename
  1280. # This makes the filename and the id consistent
  1281. #if queueJson['post'].get('id'):
  1282. # queueJson['post']['id']=queueJson['id']
  1283. if receiveUndo(session, \
  1284. baseDir,httpPrefix,port, \
  1285. sendThreads,postLog, \
  1286. cachedWebfingers,
  1287. personCache, \
  1288. queueJson['post'], \
  1289. federationList, \
  1290. debug, \
  1291. acceptedCaps=["inbox:write","objects:read"]):
  1292. if debug:
  1293. print('DEBUG: Undo accepted from '+keyId)
  1294. os.remove(queueFilename)
  1295. queue.pop(0)
  1296. continue
  1297. if debug:
  1298. print('DEBUG: checking for follow requests')
  1299. if receiveFollowRequest(session, \
  1300. baseDir,httpPrefix,port, \
  1301. sendThreads,postLog, \
  1302. cachedWebfingers,
  1303. personCache, \
  1304. queueJson['post'], \
  1305. federationList, \
  1306. debug,projectVersion, \
  1307. acceptedCaps=["inbox:write","objects:read"]):
  1308. os.remove(queueFilename)
  1309. queue.pop(0)
  1310. if debug:
  1311. print('DEBUG: Follow activity for '+keyId+' removed from accepted from queue')
  1312. continue
  1313. else:
  1314. if debug:
  1315. print('DEBUG: No follow requests')
  1316. if receiveAcceptReject(session, \
  1317. baseDir,httpPrefix,domain,port, \
  1318. sendThreads,postLog, \
  1319. cachedWebfingers, \
  1320. personCache, \
  1321. queueJson['post'], \
  1322. federationList, \
  1323. debug):
  1324. if debug:
  1325. print('DEBUG: Accept/Reject received from '+keyId)
  1326. os.remove(queueFilename)
  1327. queue.pop(0)
  1328. continue
  1329. if receiveUpdate(session, \
  1330. baseDir,httpPrefix, \
  1331. domain,port, \
  1332. sendThreads,postLog, \
  1333. cachedWebfingers, \
  1334. personCache, \
  1335. queueJson['post'], \
  1336. federationList, \
  1337. debug):
  1338. if debug:
  1339. print('DEBUG: Update accepted from '+keyId)
  1340. os.remove(queueFilename)
  1341. queue.pop(0)
  1342. continue
  1343. # get recipients list
  1344. recipientsDict,recipientsDictFollowers= \
  1345. inboxPostRecipients(baseDir,queueJson['post'], \
  1346. httpPrefix,domain,port,debug)
  1347. if len(recipientsDict.items())==0 and \
  1348. len(recipientsDictFollowers.items())==0:
  1349. if debug:
  1350. pprint(queueJson['post'])
  1351. print('DEBUG: no recipients were resolved for post arriving in inbox')
  1352. os.remove(queueFilename)
  1353. queue.pop(0)
  1354. continue
  1355. # if there are only a small number of followers then process them as if they
  1356. # were specifically addresses to particular accounts
  1357. noOfFollowItems=len(recipientsDictFollowers.items())
  1358. if noOfFollowItems>0:
  1359. if noOfFollowItems<5:
  1360. if debug:
  1361. print('DEBUG: moving '+str(noOfFollowItems)+ \
  1362. ' inbox posts addressed to followers')
  1363. for handle,postItem in recipientsDictFollowers.items():
  1364. recipientsDict[handle]=postItem
  1365. recipientsDictFollowers={}
  1366. recipientsList=[recipientsDict,recipientsDictFollowers]
  1367. if debug:
  1368. print('*************************************')
  1369. print('Resolved recipients list:')
  1370. pprint(recipientsDict)
  1371. print('Resolved followers list:')
  1372. pprint(recipientsDictFollowers)
  1373. print('*************************************')
  1374. if queueJson['post'].get('capability'):
  1375. if not isinstance(queueJson['post']['capability'], list):
  1376. if debug:
  1377. print('DEBUG: capability on post should be a list')
  1378. os.remove(queueFilename)
  1379. queue.pop(0)
  1380. continue
  1381. # Copy any posts addressed to followers into the shared inbox
  1382. # this avoid copying file multiple times to potentially many
  1383. # individual inboxes
  1384. # This obviously bypasses object capabilities and so
  1385. # any checking will needs to be handled at the time when inbox
  1386. # GET happens on individual accounts.
  1387. # See posts.py/createBoxBase
  1388. if len(recipientsDictFollowers)>0:
  1389. sharedInboxPostFilename=queueJson['destination'].replace(inboxHandle,inboxHandle)
  1390. if not os.path.isfile(sharedInboxPostFilename):
  1391. with open(sharedInboxPostFilename, 'w') as fp:
  1392. commentjson.dump(queueJson['post'],fp,indent=4, \
  1393. sort_keys=False)
  1394. # for posts addressed to specific accounts
  1395. for handle,capsId in recipientsDict.items():
  1396. destination=queueJson['destination'].replace(inboxHandle,handle)
  1397. # check that capabilities are accepted
  1398. if queueJson['post'].get('capability'):
  1399. capabilityIdList=queueJson['post']['capability']
  1400. # does the capability id list within the post contain the id
  1401. # of the recipient with this handle?
  1402. # Here the capability id begins with the handle, so this could also
  1403. # be matched separately, but it's probably not necessary
  1404. if capsId in capabilityIdList:
  1405. inboxAfterCapabilities(session,keyId,handle, \
  1406. queueJson['post'], \
  1407. baseDir,httpPrefix, \
  1408. sendThreads,postLog, \
  1409. cachedWebfingers, \
  1410. personCache,queue,domain, \
  1411. port,useTor, \
  1412. federationList,ocapAlways, \
  1413. debug,acceptedCaps, \
  1414. queueFilename,destination, \
  1415. maxReplies,allowDeletion)
  1416. else:
  1417. if debug:
  1418. print('DEBUG: object capabilities check has failed')
  1419. pprint(queueJson['post'])
  1420. else:
  1421. if not ocapAlways:
  1422. inboxAfterCapabilities(session,keyId,handle, \
  1423. queueJson['post'], \
  1424. baseDir,httpPrefix, \
  1425. sendThreads,postLog, \
  1426. cachedWebfingers, \
  1427. personCache,queue,domain, \
  1428. port,useTor, \
  1429. federationList,ocapAlways, \
  1430. debug,acceptedCaps, \
  1431. queueFilename,destination, \
  1432. maxReplies,allowDeletion)
  1433. if debug:
  1434. pprint(queueJson['post'])
  1435. print('No capability list within post')
  1436. print('ocapAlways: '+str(ocapAlways))
  1437. print('DEBUG: object capabilities check failed')
  1438. if debug:
  1439. print('DEBUG: Queue post accepted')
  1440. os.remove(queueFilename)
  1441. queue.pop(0)